class RFC2822DatetimeFilter(Filter):
"""
Given an event field with a RFC2822-compatible datetime string, convert the
string to a datetime.datetime and store it as the event ts.
"""
implements(IFilter)
def configure(self, section):
self._fieldname = section.getString('source field', 'rfc2822_date')
self._expected = section.getBoolean('expects source', False)
self._guaranteed = section.getBoolean('guarantees source', False)
self._contract = Contract()
self._contract.addAssertion( unicode(self._fieldname), u'text',
expects=self._expected, guarantees=self._guaranteed, ephemeral=True)
self._assertion = getattr(self._contract, unicode('field_' + self._fieldname))
self._contract.sign()
def getContract(self):
return self._contract
def filter(self, event):
try:
ts = email.utils.parsedate_tz(event[self._assertion])
event.ts = datetime.datetime.fromtimestamp(time.mktime(ts))
return event
except Exception, e:
raise FilterError("failed to update ts: %s" % e)
def test_create_contract(self):
contract = Contract()
# validate initial fields
self.failUnless(hasattr(contract, 'field_input') == True)
self.failUnless(contract.field_input.fieldname == 'input')
self.failUnless(contract.field_input.fieldtype == IdentityField)
self.failUnless(contract.field_input.guarantees == True)
self.failUnless(contract.field_input.ephemeral == False)
self.failUnless(hasattr(contract, 'field_hostname') == True)
self.failUnless(contract.field_hostname.fieldname == 'hostname')
self.failUnless(contract.field_hostname.fieldtype == IdentityField)
self.failUnless(contract.field_hostname.guarantees == True)
self.failUnless(contract.field_hostname.ephemeral == False)
self.failUnless(hasattr(contract, 'field_message') == True)
self.failUnless(contract.field_message.fieldname == 'message')
self.failUnless(contract.field_message.fieldtype == TextField)
self.failUnless(contract.field_message.guarantees == True)
self.failUnless(contract.field_message.ephemeral == False)
# add a custom field
contract.addAssertion('test', IdentityField, guarantees=True, ephemeral=False)
self.failUnless(hasattr(contract, 'field_test') == True)
self.failUnless(contract.field_test.fieldname == 'test')
self.failUnless(contract.field_test.fieldtype == IdentityField)
self.failUnless(contract.field_test.guarantees == True)
self.failUnless(contract.field_test.ephemeral == False)
def test_check_contract(self):
contract = Contract()
contract.addAssertion('test', IdentityField, guarantees=True, ephemeral=False)
# test len method
self.failUnless(len(contract) == 4)
# test iter method
for asrt in contract:
self.failUnless(isinstance(asrt, Assertion))
self.failUnlessIn(asrt.fieldname, ('input','hostname','message','test'))
# test field iteration method
for fieldname,fieldtype in contract.fields():
self.failUnlessIn(fieldname, ('input','hostname','message','test'))
class SyslogFilter(Filter):
implements(IFilter)
def configure(self, section):
self._linematcher = re.compile(
r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)'
)
self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$')
self._contract = Contract()
self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False)
self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False)
self._contract.sign()
def getContract(self):
return self._contract
def filter(self, event):
# split the line into timestamp, hostname, and message
m = self._linematcher.match(event[self._contract.field_message])
if m == None:
raise FilterError("[filter:%s] line is not in syslog format" %
self.name)
ts, hostname, msg = m.group('ts', 'hostname', 'msg')
if ts == None or hostname == None or msg == None:
raise FilterError("[filter:%s] line is not in syslog format" %
self.name)
# parse the timestamp
try:
event.ts = dateutil.parser.parse(ts)
except Exception, e:
raise FilterError("[filter:%s] failed to parse ts '%s': %s" %
(self.name, ts, e))
event[self._contract.field_hostname] = hostname
# split the message into tag and content
tag, content = msg.split(' ', 1)
m = self._tagmatcher.match(tag)
if m == None:
raise FilterError("[filter:%s] line has an invalid tag" %
self.name)
data = m.groups()
if data[0] != None and data[1] != None:
event[self._contract.field_syslog_tag] = data[0]
event[self._contract.field_syslog_pid] = int(data[1])
elif data[2] != None:
event[self._contract.field_syslog_tag] = data[2]
else:
raise FilterError("[filter:%s] line has an invalid tag" %
self.name)
event[self._contract.field_message] = content
return event
class SyslogFilter(Filter):
implements(IFilter)
def configure(self, section):
self._linematcher = re.compile(r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)')
self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$')
self._contract = Contract()
self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False)
self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False)
self._contract.sign()
def getContract(self):
return self._contract
def filter(self, event):
# split the line into timestamp, hostname, and message
m = self._linematcher.match(event[self._contract.field_message])
if m == None:
raise FilterError("[filter:%s] line is not in syslog format" % self.name)
ts,hostname,msg = m.group('ts','hostname','msg')
if ts == None or hostname == None or msg == None:
raise FilterError("[filter:%s] line is not in syslog format" % self.name)
# parse the timestamp
try:
event.ts = dateutil.parser.parse(ts)
except Exception, e:
raise FilterError("[filter:%s] failed to parse ts '%s': %s" % (self.name, ts, e))
event[self._contract.field_hostname] = hostname
# split the message into tag and content
tag,content = msg.split(' ', 1)
m = self._tagmatcher.match(tag)
if m == None:
raise FilterError("[filter:%s] line has an invalid tag" % self.name)
data = m.groups()
if data[0] != None and data[1] != None:
event[self._contract.field_syslog_tag] = data[0]
event[self._contract.field_syslog_pid] = int(data[1])
elif data[2] != None:
event[self._contract.field_syslog_tag] = data[2]
else:
raise FilterError("[filter:%s] line has an invalid tag" % self.name)
event[self._contract.field_message] = content
return event
class SyslogDatetimeFilter(Filter):
"""
Given an event field with a syslog-compatible datetime string, convert the
string to a datetime.datetime and store as the event ts.
"""
implements(IFilter)
def configure(self, section):
self._fieldname = section.getString('source field', 'syslog_date')
self._expected = section.getBoolean('expects source', False)
self._guaranteed = section.getBoolean('guarantees source', False)
self._contract = Contract()
self._contract.addAssertion(unicode(self._fieldname), u'text',
expects=self._expected, guarantees=self._guaranteed, ephemeral=True)
self._assertion = getattr(self._contract, unicode('field_' + self._fieldname))
self._contract.sign()
def getContract(self):
return self._contract
def filter(self, event):
try:
date = event[self._assertion]
# if there is no leading zero in front of the day, then add it
if date[4] == ' ':
date = list(date)
date[4] = '0'
date = ''.join(date)
# append the year to the date string
date += ' %i' % time.localtime()[0]
# parse the date string into a struct_time
ts = time.strptime(date, "%b %d %H:%M:%S %Y")
event.ts = datetime.datetime.fromtimestamp(time.mktime(ts))
return event
except Exception, e:
raise FilterError("failed to update ts: %s" % e)
class DatetimeExpanderFilter(Filter):
"""
Expand the event ts into separate fields for each datetime component.
"""
implements(IFilter)
def configure(self, section):
self._contract = Contract()
self._contract.addAssertion(u'dt_year', u'int', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'dt_month', u'int', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'dt_day', u'int', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'dt_hour', u'int', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'dt_minute', u'int', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'dt_second', u'int', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'dt_weekday', u'int', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'dt_yearday', u'int', expects=False, guarantees=True, ephemeral=False)
self._contract.sign()
def getContract(self):
return self._contract
def filter(self, event):
try:
tm = event.ts.timetuple()
event[self._contract.field_dt_year] = tm.tm_year
event[self._contract.field_dt_month] = tm.tm_mon
event[self._contract.field_dt_day] = tm.tm_mday
event[self._contract.field_dt_hour] = tm.tm_hour
event[self._contract.field_dt_minute] = tm.tm_min
event[self._contract.field_dt_second] = tm.tm_sec
event[self._contract.field_dt_weekday] = tm.tm_wday
event[self._contract.field_dt_yearday] = tm.tm_yday
return event
except Exception, e:
raise FilterError("failed to expand ts: %s" % e)
def test_sign_contract(self):
contract = Contract()
contract.addAssertion('test', IdentityField, guarantees=True, ephemeral=False)
# sign the contract, no more modifications allowed
contract.sign()
self.failUnlessRaises(Exception, contract.addAssertion, 'fails', TextField, guarantees=True, ephemeral=False)
class ApacheCombinedFilter(Filter):
implements(IFilter)
def configure(self, section):
self._regex = re.compile(r'''
(?P<remotehost>[\d.]+)\
(?P<remotelog>\S+)\
(?P<remoteuser>\S+)\
\[(?P<date>[\w:/]+\s[+\-]\d{4})\]\
\"(?P<request>.+?)\"\
(?P<status>\d{3})\
(?P<byteswritten>\d+)\
\"(?P<referrer>[^\"]+)\"\
\"(?P<useragent>[^\"]+)\"''',
re.VERBOSE
)
self._contract = Contract()
self._contract.addAssertion(u'remotehost', u'text', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'remotelog', u'literal', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'remoteuser', u'literal', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'request', u'text', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'status', u'int', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'byteswritten', u'int', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'referrer', u'text', expects=False, guarantees=True, ephemeral=False)
self._contract.addAssertion(u'useragent', u'text', expects=False, guarantees=True, ephemeral=False)
self._contract.sign()
def getContract(self):
return self._contract
def filter(self, event):
line = event[self._contract.message]
m = self._regex.match(line)
if m == None:
raise FilterError("incoming line '%s' didn't match regex" % line)
date = m.group('date')
if date == None:
raise FilterError("regex did not match 'date'")
# parse the timestamp
try:
event.ts = dateutil.parser.parse(date, dayfirst=True, fuzzy=True)
except Exception, e:
raise FilterError("failed to parse date '%s': %s" % (date, e))
# extract each field
for assertion in self._contract:
if assertion.fieldname in ('message', 'hostname', 'input'):
continue
value = m.group(assertion.fieldname)
if value == None:
raise FilterError("regex did not match '%s'" % assertion.fieldname)
if assertion.fieldtype == u'int':
value = int(value)
event[assertion] = value
return event
class NagiosFilter(Filter):
implements(IFilter)
def configure(self, section):
self._contract = Contract()
self._contract.addAssertion(u'nagios_evtype', u'literal', guarantees=True)
self._contract.addAssertion(u'nagios_host', u'text', guarantees=False)
self._contract.addAssertion(u'nagios_service', u'text', guarantees=False)
self._contract.addAssertion(u'nagios_status', u'text', guarantees=False)
self._contract.addAssertion(u'nagios_state', u'text', guarantees=False)
self._contract.addAssertion(u'nagios_attempt', u'int', guarantees=False)
self._contract.sign()
def getContract(self):
return self._contract
def filter(self, event):
line = event[self._contract.field_message]
# all lines should start with '['
if line[0] != '[':
raise FilterError("incoming line '%s' didn't start with timestamp" % line)
# parse the event timestamp
try:
ts,line = line[1:].split(']', 1)
except:
raise FilterError("incoming line '%s' didn't start with timestamp" % line)
try:
event.ts = datetime.fromtimestamp(float(ts))
except Exception, e:
raise FilterError("%s cannot be converted into a timestamp: %s" % (ts, e))
# determine the event type
try:
evtype,line = line.split(':', 1)
evtype = evtype.strip()
except:
raise StopFiltering()
# set the nagios_event type field
event[self._contract.field_nagios_evtype] = evtype
# parse the rest of the line
if evtype == 'HOST ALERT':
return self._hostAlert(line, event)
if evtype == 'SERVICE ALERT':
return self._serviceAlert(line, event)
if evtype == 'Error':
return self._error(line, event)
if evtype == 'Warning':
return self._warning(line, event)
raise StopFiltering()
class NagiosFilter(Filter):
implements(IFilter)
def configure(self, section):
self._contract = Contract()
self._contract.addAssertion(u'nagios_evtype',
u'literal',
guarantees=True)
self._contract.addAssertion(u'nagios_host', u'text', guarantees=False)
self._contract.addAssertion(u'nagios_service',
u'text',
guarantees=False)
self._contract.addAssertion(u'nagios_status',
u'text',
guarantees=False)
self._contract.addAssertion(u'nagios_state', u'text', guarantees=False)
self._contract.addAssertion(u'nagios_attempt',
u'int',
guarantees=False)
self._contract.sign()
def getContract(self):
return self._contract
def filter(self, event):
line = event[self._contract.field_message]
# all lines should start with '['
if line[0] != '[':
raise FilterError(
"incoming line '%s' didn't start with timestamp" % line)
# parse the event timestamp
try:
ts, line = line[1:].split(']', 1)
except:
raise FilterError(
"incoming line '%s' didn't start with timestamp" % line)
try:
event.ts = datetime.fromtimestamp(float(ts))
except Exception, e:
raise FilterError("%s cannot be converted into a timestamp: %s" %
(ts, e))
# determine the event type
try:
evtype, line = line.split(':', 1)
evtype = evtype.strip()
except:
raise StopFiltering()
# set the nagios_event type field
event[self._contract.field_nagios_evtype] = evtype
# parse the rest of the line
if evtype == 'HOST ALERT':
return self._hostAlert(line, event)
if evtype == 'SERVICE ALERT':
return self._serviceAlert(line, event)
if evtype == 'Error':
return self._error(line, event)
if evtype == 'Warning':
return self._warning(line, event)
raise StopFiltering()
class ApacheCommonFilter(Filter):
implements(IFilter)
def configure(self, section):
self._regex = re.compile(
r'''
(?P<remotehost>[\d.]+)\
(?P<remotelog>\S+)\
(?P<remoteuser>\S+)\
\[(?P<date>[\w:/]+\s[+\-]\d{4})\]\
\"(?P<request>.+?)\"\
(?P<status>\d{3})\
(?P<byteswritten>\d+)''', re.VERBOSE)
self._contract = Contract()
self._contract.addAssertion(u'remotehost',
u'text',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.addAssertion(u'remotelog',
u'text',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.addAssertion(u'remoteuser',
u'literal',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.addAssertion(u'request',
u'literal',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.addAssertion(u'status',
u'int',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.addAssertion(u'byteswritten',
u'int',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.sign()
def getContract(self):
return self._contract
def filter(self, event):
line = event[self._contract.field_message]
m = self._regex.match(line)
if m == None:
raise FilterError("incoming line '%s' didn't match regex" % line)
date = m.group('date')
if date == None:
raise FilterError("regex did not match 'date'")
# parse the timestamp
try:
event.ts = dateutil.parser.parse(date, dayfirst=True, fuzzy=True)
except Exception, e:
raise FilterError("failed to parse date '%s': %s" % (date, e))
# extract each field
for assertion in self._contract:
if assertion.fieldname in ('message', 'hostname', 'input'):
continue
value = m.group(assertion.fieldname)
if value == None:
raise FilterError("regex did not match '%s'" %
assertion.fieldname)
if assertion.fieldtype == u'int':
value = int(value)
event[assertion] = value
return event
