def parse_rfc3164(self, line):
m = self.pattern_rfc3164.match(line)
if m is not None:
with agn.Ignore(ValueError, KeyError):
(month_name, day, hour, minute, second, server, service,
content) = m.groups()
month = {
'Jan': 1,
'Feb': 2,
'Mar': 3,
'Apr': 4,
'May': 5,
'Jun': 6,
'Jul': 7,
'Aug': 8,
'Sep': 9,
'Oct': 10,
'Nov': 11,
'Dec': 12
}[month_name]
now = datetime.datetime.now()
year = now.year
if now.month < month:
year -= 1
return self.new_info(timestamp=datetime.datetime(
year, month, int(day), int(hour), int(minute),
int(second)),
server=server,
service=service,
content=content)
return None
def __iter__(self):
self.open()
dbkey = self.db.firstkey()
while dbkey is not None:
with agn.Ignore(ValueError):
(section, key) = dbkey.split(':', 1)
yield (section, key, self.get(section, key))
dbkey = self.db.nextkey(dbkey)
def remove_from_queue (self, entry): with agn.Ignore (KeyError): removed_entry = entry try: del self.in_queue[entry.statusID] except AttributeError: removed_entry = self.in_queue[entry] del self.in_queue[entry] self.log (agn.LV_DEBUG, '%s: removed from queue' % removed_entry.name)
def expire(self, created=None, updated=None):
created = self.unit.parse(created, created)
updated = self.unit.parse(updated, updated)
if created is not None or updated is not None:
self.open()
max_count = 10000
agn.log(agn.LV_INFO, 'expire', 'Expire old tracking entries')
stats = collections.defaultdict(int)
now = int(time.time())
while True:
collect = {}
dbkey = self.db.firstkey()
while dbkey is not None:
with agn.Ignore(ValueError):
(section, key) = dbkey.split(':', 1)
value = self.get(section, key)
if value:
for (expiration, timestamp_key) in [
(created, self.key_created),
(updated, self.key_updated)
]:
if expiration is not None:
try:
if value[
timestamp_key] + expiration < now:
collect[(section, key)] = True
stats[section] += 1
break
except KeyError:
collect[(section, key)] = False
if len(collect) >= max_count:
break
dbkey = self.db.nextkey(dbkey)
if collect:
agn.log(agn.LV_INFO, 'expire',
'Process %d tracking entries' % len(collect))
for ((section, key), to_delete) in collect.iteritems():
if to_delete:
self.delete(section, key)
else:
self.put(section, key, self.get(section, key))
else:
break
agn.log(
agn.LV_INFO, 'expire',
'Expiration finished (%s), reorganize database' %
(agn.Stream(stats.iteritems()).map(
lambda kv: '%s: %d' % kv).join(', ') if stats else '-', ))
self.db.reorganize()
agn.log(agn.LV_INFO, 'expire', 'Reorganization finished')
self.close()
def parse_rfc5424(self, line):
m = self.pattern_rfc5424.match(line)
if m is not None:
with agn.Ignore(ValueError):
(year, month, day, hour, minute, second, fraction, timezone, _,
server, service, content) = m.groups()
microseconds = int(float(fraction) * 1000 *
1000) if fraction else 0
return self.new_info(timestamp=datetime.datetime(
int(year), int(month), int(day), int(hour), int(minute),
int(second), microseconds),
server=server,
service=service,
content=content)
return None
def scan(self):
global term
self.expireTracker()
try:
fp = agn.Filepos(self.maillog, self.saveFile, checkpoint=1000)
except agn.error as e:
agn.log(
agn.LV_INFO, 'main',
'Unable to open %s: %s, try to gain access' %
(self.maillog, e))
n = agn.call([agn.mkpath(agn.base, 'bin', 'smctrl'), 'logaccess'])
if n != 0:
agn.log(agn.LV_ERROR, 'main',
'Failed to gain access to %s (%d)' % (self.maillog, n))
with agn.Ignore(OSError):
st = os.stat(self.saveFile)
if st.st_size == 0:
agn.log(agn.LV_ERROR, 'main',
'Remove corrupt empty file %s' % self.saveFile)
os.unlink(self.saveFile)
return
self.mtrack.open()
try:
sp = SyslogParser()
for line in fp:
try:
info = sp(line)
if info is not None:
if not self.parse(info, line):
agn.log(agn.LV_WARNING, 'scan',
'Unparsable line: %s (%r)' % (line, info))
else:
agn.log(agn.LV_WARNING, 'scan',
'Unparsable format: %s' % line)
except Exception as e:
agn.logexc(agn.LV_ERROR, 'scan',
'Failed to parse line: %s: %s' % (line, e))
if term:
break
finally:
fp.close()
self.mtrack.close()
self.processCompleted()
def delete(self, section, key):
self.open()
with agn.Ignore(KeyError):
del self.db[self.key(section, key)]