Example #1
0
 def test_create_contract(self):
     contract = Contract()
     # validate initial fields
     self.failUnless(hasattr(contract, 'field_input') == True)
     self.failUnless(contract.field_input.fieldname == 'input')
     self.failUnless(contract.field_input.fieldtype == IdentityField)
     self.failUnless(contract.field_input.guarantees == True)
     self.failUnless(contract.field_input.ephemeral == False)
     self.failUnless(hasattr(contract, 'field_hostname') == True)
     self.failUnless(contract.field_hostname.fieldname == 'hostname')
     self.failUnless(contract.field_hostname.fieldtype == IdentityField)
     self.failUnless(contract.field_hostname.guarantees == True)
     self.failUnless(contract.field_hostname.ephemeral == False)
     self.failUnless(hasattr(contract, 'field_message') == True)
     self.failUnless(contract.field_message.fieldname == 'message')
     self.failUnless(contract.field_message.fieldtype == TextField)
     self.failUnless(contract.field_message.guarantees == True)
     self.failUnless(contract.field_message.ephemeral == False)
     # add a custom field
     contract.addAssertion('test', IdentityField, guarantees=True, ephemeral=False)
     self.failUnless(hasattr(contract, 'field_test') == True)
     self.failUnless(contract.field_test.fieldname == 'test')
     self.failUnless(contract.field_test.fieldtype == IdentityField)
     self.failUnless(contract.field_test.guarantees == True)
     self.failUnless(contract.field_test.ephemeral == False)
Example #2
0
class RFC2822DatetimeFilter(Filter):
    """
    Given an event field with a RFC2822-compatible datetime string, convert the
    string to a datetime.datetime and store it as the event ts.
    """

    implements(IFilter)

    def configure(self, section):
        self._fieldname = section.getString('source field', 'rfc2822_date')
        self._expected = section.getBoolean('expects source', False)
        self._guaranteed = section.getBoolean('guarantees source', False)
        self._contract = Contract()
        self._contract.addAssertion( unicode(self._fieldname), u'text',
            expects=self._expected, guarantees=self._guaranteed, ephemeral=True)
        self._assertion = getattr(self._contract, unicode('field_' + self._fieldname))
        self._contract.sign()

    def getContract(self):
        return self._contract

    def filter(self, event):
        try:
            ts = email.utils.parsedate_tz(event[self._assertion])
            event.ts = datetime.datetime.fromtimestamp(time.mktime(ts))
            return event
        except Exception, e:
            raise FilterError("failed to update ts: %s" %  e)
Example #3
0
 def configure(self, section):
     self._fieldname = section.getString('source field', 'syslog_date')
     self._expected = section.getBoolean('expects source', False)
     self._guaranteed = section.getBoolean('guarantees source', False)
     self._contract = Contract()
     self._contract.addAssertion(unicode(self._fieldname), u'text',
         expects=self._expected, guarantees=self._guaranteed, ephemeral=True)
     self._assertion = getattr(self._contract, unicode('field_' + self._fieldname))
     self._contract.sign()
Example #4
0
 def configure(self, section):
     self._linematcher = re.compile(
         r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)'
     )
     self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$')
     self._contract = Contract()
     self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False)
     self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False)
     self._contract.sign()
Example #5
0
 def configure(self, section):
     self._contract = Contract()
     self._contract.addAssertion(u'dt_year', u'int', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'dt_month', u'int', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'dt_day', u'int', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'dt_hour', u'int', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'dt_minute', u'int', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'dt_second', u'int', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'dt_weekday', u'int', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'dt_yearday', u'int', expects=False, guarantees=True, ephemeral=False)
     self._contract.sign()
Example #6
0
 def configure(self, section):
     self._regex = re.compile(
         r'''
             (?P<remotehost>[\d.]+)\ 
             (?P<remotelog>\S+)\ 
             (?P<remoteuser>\S+)\ 
             \[(?P<date>[\w:/]+\s[+\-]\d{4})\]\ 
             \"(?P<request>.+?)\"\ 
             (?P<status>\d{3})\ 
             (?P<byteswritten>\d+)\ 
             \"(?P<referrer>[^\"]+)\"\ 
             \"(?P<useragent>[^\"]+)\"''', re.VERBOSE)
     self._contract = Contract()
     self._contract.addAssertion(u'remotehost',
                                 u'text',
                                 expects=False,
                                 guarantees=True,
                                 ephemeral=False)
     self._contract.addAssertion(u'remotelog',
                                 u'literal',
                                 expects=False,
                                 guarantees=True,
                                 ephemeral=False)
     self._contract.addAssertion(u'remoteuser',
                                 u'literal',
                                 expects=False,
                                 guarantees=True,
                                 ephemeral=False)
     self._contract.addAssertion(u'request',
                                 u'text',
                                 expects=False,
                                 guarantees=True,
                                 ephemeral=False)
     self._contract.addAssertion(u'status',
                                 u'int',
                                 expects=False,
                                 guarantees=True,
                                 ephemeral=False)
     self._contract.addAssertion(u'byteswritten',
                                 u'int',
                                 expects=False,
                                 guarantees=True,
                                 ephemeral=False)
     self._contract.addAssertion(u'referrer',
                                 u'text',
                                 expects=False,
                                 guarantees=True,
                                 ephemeral=False)
     self._contract.addAssertion(u'useragent',
                                 u'text',
                                 expects=False,
                                 guarantees=True,
                                 ephemeral=False)
     self._contract.sign()
Example #7
0
 def test_check_contract(self):
     contract = Contract()
     contract.addAssertion('test', IdentityField, guarantees=True, ephemeral=False)
     # test len method
     self.failUnless(len(contract) == 4)
     # test iter method
     for asrt in contract:
         self.failUnless(isinstance(asrt, Assertion))
         self.failUnlessIn(asrt.fieldname, ('input','hostname','message','test'))
     # test field iteration method
     for fieldname,fieldtype in contract.fields():
         self.failUnlessIn(fieldname, ('input','hostname','message','test'))
Example #8
0
 def configure(self, section):
     self._linematcher = re.compile(r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)')
     self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$')
     self._contract = Contract()
     self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False)
     self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False)
     self._contract.sign()
Example #9
0
 def configure(self, section):
     self._contract = Contract()
     self._contract.addAssertion(u'nagios_evtype',
                                 u'literal',
                                 guarantees=True)
     self._contract.addAssertion(u'nagios_host', u'text', guarantees=False)
     self._contract.addAssertion(u'nagios_service',
                                 u'text',
                                 guarantees=False)
     self._contract.addAssertion(u'nagios_status',
                                 u'text',
                                 guarantees=False)
     self._contract.addAssertion(u'nagios_state', u'text', guarantees=False)
     self._contract.addAssertion(u'nagios_attempt',
                                 u'int',
                                 guarantees=False)
     self._contract.sign()
Example #10
0
 def configure(self, section):
     self._contract = Contract()
     self._contract.addAssertion(u'nagios_evtype', u'literal', guarantees=True)
     self._contract.addAssertion(u'nagios_host', u'text', guarantees=False)
     self._contract.addAssertion(u'nagios_service', u'text', guarantees=False)
     self._contract.addAssertion(u'nagios_status', u'text', guarantees=False)
     self._contract.addAssertion(u'nagios_state', u'text', guarantees=False)
     self._contract.addAssertion(u'nagios_attempt', u'int', guarantees=False)
     self._contract.sign()
Example #11
0
 def test_dynamic_schema(self):
     index = self.output.getIndex()
     schema = index.getSchema()
     contract = Contract().sign()
     ts,offset,message = Output_Store_Tests.test_data[0]
     event = Event(ts, offset)
     event[contract.field_message] = message
     self.output.receiveEvent(event)
     for fieldname,fieldtype,fieldvalue in event:
         self.assertTrue(schema.hasField(fieldname, fieldtype))
Example #12
0
class SyslogDatetimeFilter(Filter):
    """
    Given an event field with a syslog-compatible datetime string, convert the
    string to a datetime.datetime and store as the event ts.
    """

    implements(IFilter)

    def configure(self, section):
        self._fieldname = section.getString('source field', 'syslog_date')
        self._expected = section.getBoolean('expects source', False)
        self._guaranteed = section.getBoolean('guarantees source', False)
        self._contract = Contract()
        self._contract.addAssertion(unicode(self._fieldname), u'text',
            expects=self._expected, guarantees=self._guaranteed, ephemeral=True)
        self._assertion = getattr(self._contract, unicode('field_' + self._fieldname))
        self._contract.sign()

    def getContract(self):
        return self._contract

    def filter(self, event):
        try:
            date = event[self._assertion]
            # if there is no leading zero in front of the day, then add it
            if date[4] == ' ':
                date = list(date)
                date[4] = '0'
                date = ''.join(date)
            # append the year to the date string
            date += ' %i' % time.localtime()[0]
            # parse the date string into a struct_time
            ts = time.strptime(date, "%b %d %H:%M:%S %Y")
            event.ts = datetime.datetime.fromtimestamp(time.mktime(ts))
            return event
        except Exception, e:
            raise FilterError("failed to update ts: %s" %  e)
Example #13
0
class SyslogFilter(Filter):

    implements(IFilter)

    def configure(self, section):
        self._linematcher = re.compile(
            r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)'
        )
        self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$')
        self._contract = Contract()
        self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False)
        self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False)
        self._contract.sign()

    def getContract(self):
        return self._contract

    def filter(self, event):
        # split the line into timestamp, hostname, and message
        m = self._linematcher.match(event[self._contract.field_message])
        if m == None:
            raise FilterError("[filter:%s] line is not in syslog format" %
                              self.name)
        ts, hostname, msg = m.group('ts', 'hostname', 'msg')
        if ts == None or hostname == None or msg == None:
            raise FilterError("[filter:%s] line is not in syslog format" %
                              self.name)
        # parse the timestamp
        try:
            event.ts = dateutil.parser.parse(ts)
        except Exception, e:
            raise FilterError("[filter:%s] failed to parse ts '%s': %s" %
                              (self.name, ts, e))
        event[self._contract.field_hostname] = hostname
        # split the message into tag and content
        tag, content = msg.split(' ', 1)
        m = self._tagmatcher.match(tag)
        if m == None:
            raise FilterError("[filter:%s] line has an invalid tag" %
                              self.name)
        data = m.groups()
        if data[0] != None and data[1] != None:
            event[self._contract.field_syslog_tag] = data[0]
            event[self._contract.field_syslog_pid] = int(data[1])
        elif data[2] != None:
            event[self._contract.field_syslog_tag] = data[2]
        else:
            raise FilterError("[filter:%s] line has an invalid tag" %
                              self.name)
        event[self._contract.field_message] = content
        return event
Example #14
0
 def configure(self, section):
     self._regex = re.compile(r'''
             (?P<remotehost>[\d.]+)\ 
             (?P<remotelog>\S+)\ 
             (?P<remoteuser>\S+)\ 
             \[(?P<date>[\w:/]+\s[+\-]\d{4})\]\ 
             \"(?P<request>.+?)\"\ 
             (?P<status>\d{3})\ 
             (?P<byteswritten>\d+)''',
         re.VERBOSE
         )
     self._contract = Contract()
     self._contract.addAssertion(u'remotehost', u'text', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'remotelog', u'text', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'remoteuser', u'literal', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'request', u'literal', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'status', u'int', expects=False, guarantees=True, ephemeral=False)
     self._contract.addAssertion(u'byteswritten', u'int', expects=False, guarantees=True, ephemeral=False)
     self._contract.sign()
Example #15
0
 def test_read_write_index(self):
     contract = Contract().sign()
     for ts,offset,message in Output_Store_Tests.test_data:
         event = Event(ts, offset)
         event[contract.field_message] = message
         self.output.receiveEvent(event)
     # read back from the index
     index = self.output.getIndex()
     startId = EVID.fromDatetime(*Output_Store_Tests.test_data[0][0:2])
     endId = EVID.fromDatetime(*Output_Store_Tests.test_data[-1][0:2])
     searcher = index.newSearcher()
     try:
         npostings = searcher.postingsLength(None, None, startId, endId)
         self.assertTrue(npostings == len(Output_Store_Tests.test_data))
         i = searcher.iterPostings(None, None, startId, endId)
         postings = []
         while True:
             posting = i.nextPosting()
             if posting == (None, None, None):
                 break
             postings.append(posting)
     finally:
         searcher.close()
Example #16
0
class SyslogFilter(Filter):

    implements(IFilter)

    def configure(self, section):
        self._linematcher = re.compile(r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)')
        self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$')
        self._contract = Contract()
        self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False)
        self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False)
        self._contract.sign()

    def getContract(self):
        return self._contract

    def filter(self, event):
        # split the line into timestamp, hostname, and message
        m = self._linematcher.match(event[self._contract.field_message])
        if m == None:
            raise FilterError("[filter:%s] line is not in syslog format" % self.name)
        ts,hostname,msg = m.group('ts','hostname','msg')
        if ts == None or hostname == None or msg == None:
            raise FilterError("[filter:%s] line is not in syslog format" % self.name)
        # parse the timestamp
        try:
            event.ts = dateutil.parser.parse(ts)
        except Exception, e:
            raise FilterError("[filter:%s] failed to parse ts '%s': %s" % (self.name, ts, e))
        event[self._contract.field_hostname] = hostname
        # split the message into tag and content
        tag,content = msg.split(' ', 1)
        m = self._tagmatcher.match(tag)
        if m == None:
            raise FilterError("[filter:%s] line has an invalid tag" % self.name)
        data = m.groups()
        if data[0] != None and data[1] != None:
            event[self._contract.field_syslog_tag] = data[0]
            event[self._contract.field_syslog_pid] = int(data[1])
        elif data[2] != None:
            event[self._contract.field_syslog_tag] = data[2]
        else:
            raise FilterError("[filter:%s] line has an invalid tag" % self.name)
        event[self._contract.field_message] = content
        return event
Example #17
0
 def setUp(self):
     datadir = os.path.abspath(self.mktemp())
     os.mkdir(datadir)
     settings = _UnittestSettings()
     settings.load({
         'plugin:output:store': {
             'data directory': datadir,
         },
         'output:test': {
             'type': 'store',
         }
     })
     self.plugin = StoreOutputPlugin()
     self.plugin.configure(settings.section('plugin:output:store'))
     self.output = StoreOutput(self.plugin, 'test', MockFieldStore())
     self.output.configure(settings.section('output:test'))
     self.plugin.startService()
     self.output.startService()
     contract = Contract().sign()
     # write events to the index
     for ts, offset, message in Bier_Searching_Tests.test_data:
         event = Event(ts, offset)
         event[contract.field_message] = message
         self.output.receiveEvent(event)
Example #18
0
 def test_validate_contract_succeeds(self):
     contract = Contract().sign()
     prior = Contract().sign()
     contract.validateContract(prior)
Example #19
0
class NagiosFilter(Filter):

    implements(IFilter)

    def configure(self, section):
        self._contract = Contract()
        self._contract.addAssertion(u'nagios_evtype',
                                    u'literal',
                                    guarantees=True)
        self._contract.addAssertion(u'nagios_host', u'text', guarantees=False)
        self._contract.addAssertion(u'nagios_service',
                                    u'text',
                                    guarantees=False)
        self._contract.addAssertion(u'nagios_status',
                                    u'text',
                                    guarantees=False)
        self._contract.addAssertion(u'nagios_state', u'text', guarantees=False)
        self._contract.addAssertion(u'nagios_attempt',
                                    u'int',
                                    guarantees=False)
        self._contract.sign()

    def getContract(self):
        return self._contract

    def filter(self, event):
        line = event[self._contract.field_message]
        # all lines should start with '['
        if line[0] != '[':
            raise FilterError(
                "incoming line '%s' didn't start with timestamp" % line)
        # parse the event timestamp
        try:
            ts, line = line[1:].split(']', 1)
        except:
            raise FilterError(
                "incoming line '%s' didn't start with timestamp" % line)
        try:
            event.ts = datetime.fromtimestamp(float(ts))
        except Exception, e:
            raise FilterError("%s cannot be converted into a timestamp: %s" %
                              (ts, e))
        # determine the event type
        try:
            evtype, line = line.split(':', 1)
            evtype = evtype.strip()
        except:
            raise StopFiltering()
        # set the nagios_event type field
        event[self._contract.field_nagios_evtype] = evtype
        # parse the rest of the line
        if evtype == 'HOST ALERT':
            return self._hostAlert(line, event)
        if evtype == 'SERVICE ALERT':
            return self._serviceAlert(line, event)
        if evtype == 'Error':
            return self._error(line, event)
        if evtype == 'Warning':
            return self._warning(line, event)
        raise StopFiltering()
Example #20
0
class NagiosFilter(Filter):

    implements(IFilter)

    def configure(self, section):
        self._contract = Contract()
        self._contract.addAssertion(u'nagios_evtype', u'literal', guarantees=True)
        self._contract.addAssertion(u'nagios_host', u'text', guarantees=False)
        self._contract.addAssertion(u'nagios_service', u'text', guarantees=False)
        self._contract.addAssertion(u'nagios_status', u'text', guarantees=False)
        self._contract.addAssertion(u'nagios_state', u'text', guarantees=False)
        self._contract.addAssertion(u'nagios_attempt', u'int', guarantees=False)
        self._contract.sign()

    def getContract(self):
        return self._contract

    def filter(self, event):
        line = event[self._contract.field_message]
        # all lines should start with '['
        if line[0] != '[':
            raise FilterError("incoming line '%s' didn't start with timestamp" % line)
        # parse the event timestamp
        try:
            ts,line = line[1:].split(']', 1)
        except:
            raise FilterError("incoming line '%s' didn't start with timestamp" % line)
        try:
            event.ts = datetime.fromtimestamp(float(ts))
        except Exception, e:
            raise FilterError("%s cannot be converted into a timestamp: %s" % (ts, e))
        # determine the event type
        try:
            evtype,line = line.split(':', 1)
            evtype = evtype.strip()
        except:
            raise StopFiltering()
        # set the nagios_event type field
        event[self._contract.field_nagios_evtype] = evtype
        # parse the rest of the line
        if evtype == 'HOST ALERT':
            return self._hostAlert(line, event)
        if evtype == 'SERVICE ALERT':
            return self._serviceAlert(line, event)
        if evtype == 'Error':
            return self._error(line, event)
        if evtype == 'Warning':
            return self._warning(line, event)
        raise StopFiltering()
Example #21
0
 def configure(self, section):
     self._regex = re.compile(
         r'(?P<date>\d{6})\w+(?P<time>\d\d:\d\d\:\d\d)\w+(?P<msg>.*)')
     self._contract = Contract().sign()
Example #22
0
 def __init__(self, plugin, name, fieldstore):
     self._plugin = plugin
     self.setName(name)
     self._fieldstore = fieldstore
     self._index = None
     self._contract = Contract().sign()
Example #23
0
 def test_get_contract(self):
     contract = self.output.getContract()
     prior = Contract().sign()
     contract.validateContract(prior)
Example #24
0
class ApacheCommonFilter(Filter):

    implements(IFilter)

    def configure(self, section):
        self._regex = re.compile(
            r'''
                (?P<remotehost>[\d.]+)\ 
                (?P<remotelog>\S+)\ 
                (?P<remoteuser>\S+)\ 
                \[(?P<date>[\w:/]+\s[+\-]\d{4})\]\ 
                \"(?P<request>.+?)\"\ 
                (?P<status>\d{3})\ 
                (?P<byteswritten>\d+)''', re.VERBOSE)
        self._contract = Contract()
        self._contract.addAssertion(u'remotehost',
                                    u'text',
                                    expects=False,
                                    guarantees=True,
                                    ephemeral=False)
        self._contract.addAssertion(u'remotelog',
                                    u'text',
                                    expects=False,
                                    guarantees=True,
                                    ephemeral=False)
        self._contract.addAssertion(u'remoteuser',
                                    u'literal',
                                    expects=False,
                                    guarantees=True,
                                    ephemeral=False)
        self._contract.addAssertion(u'request',
                                    u'literal',
                                    expects=False,
                                    guarantees=True,
                                    ephemeral=False)
        self._contract.addAssertion(u'status',
                                    u'int',
                                    expects=False,
                                    guarantees=True,
                                    ephemeral=False)
        self._contract.addAssertion(u'byteswritten',
                                    u'int',
                                    expects=False,
                                    guarantees=True,
                                    ephemeral=False)
        self._contract.sign()

    def getContract(self):
        return self._contract

    def filter(self, event):
        line = event[self._contract.field_message]
        m = self._regex.match(line)
        if m == None:
            raise FilterError("incoming line '%s' didn't match regex" % line)
        date = m.group('date')
        if date == None:
            raise FilterError("regex did not match 'date'")
        # parse the timestamp
        try:
            event.ts = dateutil.parser.parse(date, dayfirst=True, fuzzy=True)
        except Exception, e:
            raise FilterError("failed to parse date '%s': %s" % (date, e))
        # extract each field
        for assertion in self._contract:
            if assertion.fieldname in ('message', 'hostname', 'input'):
                continue
            value = m.group(assertion.fieldname)
            if value == None:
                raise FilterError("regex did not match '%s'" %
                                  assertion.fieldname)
            if assertion.fieldtype == u'int':
                value = int(value)
            event[assertion] = value
        return event
Example #25
0
class ApacheCombinedFilter(Filter):

    implements(IFilter)

    def configure(self, section):
        self._regex = re.compile(r'''
                (?P<remotehost>[\d.]+)\ 
                (?P<remotelog>\S+)\ 
                (?P<remoteuser>\S+)\ 
                \[(?P<date>[\w:/]+\s[+\-]\d{4})\]\ 
                \"(?P<request>.+?)\"\ 
                (?P<status>\d{3})\ 
                (?P<byteswritten>\d+)\ 
                \"(?P<referrer>[^\"]+)\"\ 
                \"(?P<useragent>[^\"]+)\"''',
            re.VERBOSE
            )
        self._contract = Contract()
        self._contract.addAssertion(u'remotehost', u'text', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'remotelog', u'literal', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'remoteuser', u'literal', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'request', u'text', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'status', u'int', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'byteswritten', u'int', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'referrer', u'text', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'useragent', u'text', expects=False, guarantees=True, ephemeral=False)
        self._contract.sign()

    def getContract(self):
        return self._contract

    def filter(self, event):
        line = event[self._contract.message]
        m = self._regex.match(line)
        if m == None:
            raise FilterError("incoming line '%s' didn't match regex" % line)
        date = m.group('date')
        if date == None:
            raise FilterError("regex did not match 'date'")
        # parse the timestamp
        try:
            event.ts = dateutil.parser.parse(date, dayfirst=True, fuzzy=True)
        except Exception, e:
            raise FilterError("failed to parse date '%s': %s" % (date, e))
        # extract each field
        for assertion in self._contract:
            if assertion.fieldname in ('message', 'hostname', 'input'):
                continue
            value = m.group(assertion.fieldname)
            if value == None:
                raise FilterError("regex did not match '%s'" % assertion.fieldname)
            if assertion.fieldtype == u'int':
                value = int(value)
            event[assertion] = value
        return event
Example #26
0
 def test_sign_contract(self):
     contract = Contract()
     contract.addAssertion('test', IdentityField, guarantees=True, ephemeral=False)
     # sign the contract, no more modifications allowed
     contract.sign()
     self.failUnlessRaises(Exception, contract.addAssertion, 'fails', TextField, guarantees=True, ephemeral=False)
Example #27
0
class DatetimeExpanderFilter(Filter):
    """
    Expand the event ts into separate fields for each datetime component.
    """

    implements(IFilter)

    def configure(self, section):
        self._contract = Contract()
        self._contract.addAssertion(u'dt_year', u'int', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'dt_month', u'int', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'dt_day', u'int', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'dt_hour', u'int', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'dt_minute', u'int', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'dt_second', u'int', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'dt_weekday', u'int', expects=False, guarantees=True, ephemeral=False)
        self._contract.addAssertion(u'dt_yearday', u'int', expects=False, guarantees=True, ephemeral=False)
        self._contract.sign()

    def getContract(self):
        return self._contract

    def filter(self, event):
        try:
            tm = event.ts.timetuple()
            event[self._contract.field_dt_year] = tm.tm_year
            event[self._contract.field_dt_month] = tm.tm_mon
            event[self._contract.field_dt_day] = tm.tm_mday
            event[self._contract.field_dt_hour] = tm.tm_hour
            event[self._contract.field_dt_minute] = tm.tm_min
            event[self._contract.field_dt_second] = tm.tm_sec
            event[self._contract.field_dt_weekday] = tm.tm_wday
            event[self._contract.field_dt_yearday] = tm.tm_yday
            return event
        except Exception, e:
            raise FilterError("failed to expand ts: %s" % e)
Example #28
0
 def test_validate_contract_expects_missing_field(self):
     prior = Contract().sign()
     contract = Contract().addAssertion('test', TextField, expects=True).sign()
     self.failUnlessRaises(Exception, contract.validatesAgainst, prior)