def publish(self):
"""Publish a mbox"""
mbox = Mbox(self.config.get('mbox'))
messages = 0
if not (os.path.exists(self.config.get('dir'))):
os.mkdir(self.config.get('dir'))
message = mbox.nextMessage()
while(message != None):
msg = Message(message, self.config)
messages += 1
self.index.add(msg)
self.subscribers.add(msg)
msg.toRDF()
#msg.toHTML()
message = mbox.nextMessage()
self.index.toRDF()
self.subscribers.toRDF()
return messages
def publish(self):
"""
Publish the messages
"""
self.__createDir()
#fisrt lap
self.__parse()
#and second lap
mbox = Mbox(self.config.get('mbox'))
messages = 0
message = mbox.nextMessage()
try:
while (message != None):
messages += 1
id = message['Message-Id']
msg = self.index.getMessage(messages)
if (msg != None and msg.getMessageId() == id):
msg.setBody(message.fp.read())
msg.toRDF()
#msg.toHTML()
#self.index.delete(id)
else:
print 'Someone was wrong with message ' + str(
messages) + ' with ID ' + id + ' (' + msg.getMessageId(
) + ')'
message = mbox.nextMessage()
self.__toRDF()
if (self.config.get('foaf')):
self.subscribers.process()
self.subscribers.export()
except Exception, detail:
print str(detail)
def __parse(self):
"""
Parse mailingg list and load all
indexes into memory
"""
previous = None
mbox = Mbox(self.config.get('mbox'))
messages = 0
message = mbox.nextMessage()
while(message != None):
#fisrt load message
messages += 1
msg = Message(message, self.config)
#index it
self.index.add(msg)
self.subscribers.add(msg)
subscriber = self.subscribers.get(msg.getFromMail())
msg.setSender(subscriber)
#parent message (refactor)
inReplyTo = msg.getInReplyTo()
if (inReplyTo != None):
parent = self.index.get(inReplyTo)
if (parent != None):
msg.setParent(parent) #link child with parent
parent.addChild(msg) #and parent with child
#and previous and next by date
if (previous != None):
previous.setNextByDate(msg)
msg.setPreviousByDate(previous)
previous = msg
#and continue with next message
message = mbox.nextMessage()
self.messages = messages
def __parse(self):
"""
Parse mailingg list and load all
indexes into memory
"""
previous = None
mbox = Mbox(self.config.get('mbox'))
messages = 0
message = mbox.nextMessage()
while (message != None):
#fisrt load message
messages += 1
msg = Message(message, self.config)
#index it
self.index.add(msg)
self.subscribers.add(msg)
subscriber = self.subscribers.get(msg.getFromMail())
msg.setSender(subscriber)
#parent message (refactor)
inReplyTo = msg.getInReplyTo()
if (inReplyTo != None):
parent = self.index.get(inReplyTo)
if (parent != None):
msg.setParent(parent) #link child with parent
parent.addChild(msg) #and parent with child
#and previous and next by date
if (previous != None):
previous.setNextByDate(msg)
msg.setPreviousByDate(previous)
previous = msg
#and continue with next message
message = mbox.nextMessage()
self.messages = messages
def publish(self):
"""
Publish the messages
"""
self.__createDir()
# fisrt lap
self.__parse()
# and second lap
mbox = Mbox(self.config.get("mbox"))
messages = 0
message = mbox.nextMessage()
try:
while message != None:
messages += 1
id = message["Message-Id"]
msg = self.index.getMessage(messages)
if msg != None and msg.getMessageId() == id:
msg.setBody("FIXME")
msg.toRDF()
# msg.toHTML()
# self.index.delete(id)
else:
print "Someone was wrong with message " + str(
messages
) + " with ID " + id + " (" + msg.getMessageId() + ")"
message = mbox.nextMessage()
self.__toRDF()
self.subscribers.process()
self.subscribers.export()
except Exception, detail:
print str(detail)
def publish(self):
"""
Publish the messages
"""
self.__createDir()
#fisrt lap
self.__parse()
#and second lap
mbox = Mbox(self.config.get('mbox'))
messages = 0
message = mbox.nextMessage()
try:
while(message != None):
messages += 1
id = message['Message-Id']
msg = self.index.getMessage(messages)
if (msg != None and msg.getMessageId() == id):
msg.setBody(message.fp.read())
msg.toRDF()
#msg.toHTML()
#self.index.delete(id)
else:
print 'Someone was wrong with message ' + str(messages) + ' with ID ' + id + ' ('+msg.getMessageId()+')'
message = mbox.nextMessage()
self.__toRDF()
if (self.config.get('foaf')):
self.subscribers.process()
self.subscribers.export()
except Exception, detail:
print str(detail)
def publish(self):
"""
Publish the messages
"""
self.__createDir()
#fisrt lap
self.__parse()
#and second lap
mbox = Mbox(self.config.get('mbox'))
messages = 0
message = mbox.nextMessage()
try:
while(message != None):
messages += 1
id = message['Message-Id']
msg = self.index.get(id)
if (msg != None):
msg.setBody('FIXME')
msg.toRDF()
#msg.toHTML()
#self.index.delete(id)
else:
print id
message = mbox.nextMessage()
self.__toRDF()
self.subscribers.process()
self.subscribers.export()
except Exception, detail:
print str(detail)
def publish(self):
"""
Publish the messages
"""
self.__createDir()
#fisrt lap
self.__parse()
#and second lap
mbox = Mbox(self.config.get('mbox'))
messages = 0
message = mbox.nextMessage()
try:
while (message != None):
messages += 1
id = message['Message-Id']
msg = self.index.get(id)
if (msg != None):
msg.setBody('FIXME')
msg.toRDF()
#msg.toHTML()
#self.index.delete(id)
else:
print id
message = mbox.nextMessage()
self.__toRDF()
self.subscribers.process()
self.subscribers.export()
except Exception, detail:
print str(detail)
from mbox import Mbox
def documents_from_mails(mails):
"""Build document from mail"""
for ts, mail in mails:
if ts[3] < 0: # This bug is audacious
ts = list(ts)
ts[3] = 12
ts = tuple(ts)
yield {
'@source': 'stuff://',
'@type': 'mailadmin',
'@tags': [mail.headers['From']],
'@fields': mail.headers,
'@timestamp': time.strftime("%Y-%m-%dT%H:%M:%S", ts),
'@message': mail.body,
'id': mail.headers['Message-Id']
}
if __name__ == '__main__':
import sys
from poteau import Kibana
# Instantiate it with an url
es = ElasticSearch(sys.argv[1], timeout=240, max_retries=10)
k = Kibana(es)
# Kibana need this kind of name
emails = Mbox(sys.argv[2])
for day, size in k.index_documents('email', documents_from_mails(emails)):
print("[%s] #%i" % (day, size))