def filter(self, event):
line = event[self._contract.field_message]
m = self._regex.match(line)
if m == None:
raise FilterError("incoming line '%s' didn't match regex" % line)
date = m.group('date')
if date == None:
raise FilterError("regex did not match 'date'")
# parse the timestamp
try:
event.ts = dateutil.parser.parse(date, dayfirst=True, fuzzy=True)
except Exception, e:
raise FilterError("failed to parse date '%s': %s" % (date, e))
class SyslogFilter(Filter):
implements(IFilter)
def configure(self, section):
self._linematcher = re.compile(
r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)'
)
self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$')
self._contract = Contract()
self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False)
self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False)
self._contract.sign()
def getContract(self):
return self._contract
def filter(self, event):
# split the line into timestamp, hostname, and message
m = self._linematcher.match(event[self._contract.field_message])
if m == None:
raise FilterError("[filter:%s] line is not in syslog format" %
self.name)
ts, hostname, msg = m.group('ts', 'hostname', 'msg')
if ts == None or hostname == None or msg == None:
raise FilterError("[filter:%s] line is not in syslog format" %
self.name)
# parse the timestamp
try:
event.ts = dateutil.parser.parse(ts)
except Exception, e:
raise FilterError("[filter:%s] failed to parse ts '%s': %s" %
(self.name, ts, e))
event[self._contract.field_hostname] = hostname
# split the message into tag and content
tag, content = msg.split(' ', 1)
m = self._tagmatcher.match(tag)
if m == None:
raise FilterError("[filter:%s] line has an invalid tag" %
self.name)
data = m.groups()
if data[0] != None and data[1] != None:
event[self._contract.field_syslog_tag] = data[0]
event[self._contract.field_syslog_pid] = int(data[1])
elif data[2] != None:
event[self._contract.field_syslog_tag] = data[2]
else:
raise FilterError("[filter:%s] line has an invalid tag" %
self.name)
event[self._contract.field_message] = content
return event
def filter(self, event):
try:
ts = email.utils.parsedate_tz(event[self._assertion])
event.ts = datetime.datetime.fromtimestamp(time.mktime(ts))
return event
except Exception, e:
raise FilterError("failed to update ts: %s" % e)
def filter(self, event):
# split the line into timestamp, hostname, and message
m = self._linematcher.match(event[self._contract.field_message])
if m == None:
raise FilterError("[filter:%s] line is not in syslog format" %
self.name)
ts, hostname, msg = m.group('ts', 'hostname', 'msg')
if ts == None or hostname == None or msg == None:
raise FilterError("[filter:%s] line is not in syslog format" %
self.name)
# parse the timestamp
try:
event.ts = dateutil.parser.parse(ts)
except Exception, e:
raise FilterError("[filter:%s] failed to parse ts '%s': %s" %
(self.name, ts, e))
def filter(self, event):
line = event[self._contract.field_message]
# all lines should start with '['
if line[0] != '[':
raise FilterError(
"incoming line '%s' didn't start with timestamp" % line)
# parse the event timestamp
try:
ts, line = line[1:].split(']', 1)
except:
raise FilterError(
"incoming line '%s' didn't start with timestamp" % line)
try:
event.ts = datetime.fromtimestamp(float(ts))
except Exception, e:
raise FilterError("%s cannot be converted into a timestamp: %s" %
(ts, e))
def filter(self, fields):
if not self._infield in fields:
raise FilterError("input is missing '%s' field" % self._infield)
m = self.regex.match(fields[self._infield])
if m == None:
raise StopFiltering("input '%s' didn't match regex" %
fields[self._infield])
fields.update(m.groupdict())
return fields
def _hostAlert(self, line, event):
try:
host, status, state, attempt, detail = line.strip().split(';', 4)
event[self._contract.field_nagios_host] = host
event[self._contract.field_nagios_status] = status
event[self._contract.field_nagios_state] = state
event[self._contract.field_nagios_attempt] = int(attempt)
event[self._contract.field_message] = detail
return event
except Exception, e:
FilterError("failed to parse host alert: %s" % e)
def filter(self, event):
m = self._regex.match(event[self._contract.field_message])
# if the regex matches, then we have a timestamped event, otherwise
# keep the entire line in the message field.
if m != None:
try:
# override the default timestamp
date = "%s %s" % m.group('date', 'time')
event.ts = datetime.datetime.strptime(date, '%y%m%d %%H:%M:%S')
except Exception, e:
raise FilterError("failed to parse timestamp: %s" % e)
# put the rest of the line into the default field
event[self._contract.field_message] = m.group('msg')
def filter(self, event):
try:
tm = event.ts.timetuple()
event[self._contract.field_dt_year] = tm.tm_year
event[self._contract.field_dt_month] = tm.tm_mon
event[self._contract.field_dt_day] = tm.tm_mday
event[self._contract.field_dt_hour] = tm.tm_hour
event[self._contract.field_dt_minute] = tm.tm_min
event[self._contract.field_dt_second] = tm.tm_sec
event[self._contract.field_dt_weekday] = tm.tm_wday
event[self._contract.field_dt_yearday] = tm.tm_yday
return event
except Exception, e:
raise FilterError("failed to expand ts: %s" % e)
def filter(self, event):
try:
date = event[self._assertion]
# if there is no leading zero in front of the day, then add it
if date[4] == ' ':
date = list(date)
date[4] = '0'
date = ''.join(date)
# append the year to the date string
date += ' %i' % time.localtime()[0]
# parse the date string into a struct_time
ts = time.strptime(date, "%b %d %H:%M:%S %Y")
event.ts = datetime.datetime.fromtimestamp(time.mktime(ts))
return event
except Exception, e:
raise FilterError("failed to update ts: %s" % e)
class ApacheCommonFilter(Filter):
implements(IFilter)
def configure(self, section):
self._regex = re.compile(
r'''
(?P<remotehost>[\d.]+)\
(?P<remotelog>\S+)\
(?P<remoteuser>\S+)\
\[(?P<date>[\w:/]+\s[+\-]\d{4})\]\
\"(?P<request>.+?)\"\
(?P<status>\d{3})\
(?P<byteswritten>\d+)''', re.VERBOSE)
self._contract = Contract()
self._contract.addAssertion(u'remotehost',
u'text',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.addAssertion(u'remotelog',
u'text',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.addAssertion(u'remoteuser',
u'literal',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.addAssertion(u'request',
u'literal',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.addAssertion(u'status',
u'int',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.addAssertion(u'byteswritten',
u'int',
expects=False,
guarantees=True,
ephemeral=False)
self._contract.sign()
def getContract(self):
return self._contract
def filter(self, event):
line = event[self._contract.field_message]
m = self._regex.match(line)
if m == None:
raise FilterError("incoming line '%s' didn't match regex" % line)
date = m.group('date')
if date == None:
raise FilterError("regex did not match 'date'")
# parse the timestamp
try:
event.ts = dateutil.parser.parse(date, dayfirst=True, fuzzy=True)
except Exception, e:
raise FilterError("failed to parse date '%s': %s" % (date, e))
# extract each field
for assertion in self._contract:
if assertion.fieldname in ('message', 'hostname', 'input'):
continue
value = m.group(assertion.fieldname)
if value == None:
raise FilterError("regex did not match '%s'" %
assertion.fieldname)
if assertion.fieldtype == u'int':
value = int(value)
event[assertion] = value
return event