def fetchMails(maillist, serverNamePort="http://localhost:5984"):
#print time.clock()
ID = BMM.getAllMessageIDForMailingList(maillist, serverNamePort)
cdb.createDatabase(maillist, serverNamePort)
#print time.clock()
# fetch and insert emails in the database
n = len(ID) / numberOfThreadsCouchDB
q = Queue.Queue()
# the first in the list is also the last mail read in
if len(ID) > 0:
t = MailFetcher(maillist, serverNamePort, [ID[0]], "First", q)
t.start()
q.put(t) # put in the queue
q.join() # wait to finish
# update last element in couchdb
doc = {'_id': 'lastEmail', 'key': ID[0]}
print "last " + ID[0] + " " + str(doc)
cdb.updateDocCouchDB(doc, maillist, serverNamePort)
# create the pool for the other emails
print "Fetching the mails"
if n == 0:
n = 1
for i in range(0, len(ID), n):
if i + n + 1 >= len(ID):
t = MailFetcher(maillist, serverNamePort, ID[i + 1:len(ID)],
"Block-" + str(i + 1) + "-" + str(len(ID)), q)
else:
t = MailFetcher(maillist, serverNamePort, ID[i + 1:i + n + 1],
"Block-" + str(i + 1) + "-" + str(i + n + 1), q)
t.start()
q.put(t)
q.join()
def fetchMails(maillist, serverNamePort="http://localhost:5984"):
#print time.clock()
ID=BMM.getAllMessageIDForMailingList(maillist, serverNamePort)
cdb.createDatabase(maillist, serverNamePort)
#print time.clock()
# fetch and insert emails in the database
n=len(ID)/numberOfThreadsCouchDB
q= Queue.Queue()
# the first in the list is also the last mail read in
if len(ID)>0:
t=MailFetcher(maillist, serverNamePort, [ID[0]], "First", q)
t.start()
q.put(t) # put in the queue
q.join() # wait to finish
# update last element in couchdb
doc={'_id': 'lastEmail', 'key':ID[0]}
print "last "+ID[0]+" "+str(doc)
cdb.updateDocCouchDB(doc, maillist, serverNamePort)
# create the pool for the other emails
print "Fetching the mails"
if n==0:
n=1
for i in range(0, len(ID), n):
if i+n+1>=len(ID):
t=MailFetcher(maillist, serverNamePort, ID[i+1:len(ID)], "Block-"+str(i+1)+"-"+str(len(ID)), q)
else:
t=MailFetcher(maillist, serverNamePort, ID[i+1:i+n+1], "Block-"+str(i+1)+"-"+str(i+n+1), q)
t.start()
q.put(t)
q.join()
def run(self):
n = len(self.mailIDToProcess)
# get all the mail
self.queue.get()
for i in range(n):
#print self.name+" "+self.mailIDToProcess[i]
mail = FMM.getMailMessage(self.mailIDToProcess[i], self.maillist)
# insert the mail in couchdb
if not (mail == None):
cdb.unsafeSaveMailCouchdb(mail, self.servName)
self.queue.task_done()
print " Thread " + str(self.name) + " finished."
def run(self):
n=len(self.mailIDToProcess)
# get all the mail
self.queue.get()
for i in range(n):
#print self.name+" "+self.mailIDToProcess[i]
mail = FMM.getMailMessage(self.mailIDToProcess[i], self.maillist)
# insert the mail in couchdb
if not(mail==None):
cdb.unsafeSaveMailCouchdb(mail, self.servName)
self.queue.task_done()
print " Thread "+str(self.name)+" finished."
def getAllMessageIDForMailingList(maillist, serverNamePort="http://localhost:5984", threadsNumb=25):
global foundLast
# get all the date pages
dateList=getDateList(maillist)
messList=[]
q=Queue.Queue()
#get the last message in couchdb
lastId=cdb.getLastEmailKey(maillist, serverNamePort)
# get all the messages relatives to the first date
t=GetIDs(maillist, dateList[0], q, messList, lastId)
t.start()
q.put(t)
q.join()
# get all the rest using threads
if not(foundLast):
for i in range(1,len(dateList),threadsNumb):
for j in range(0,threadsNumb):
if i+j>=len(dateList):
break
else:
t=GetIDs(maillist, dateList[i+j], q, messList, lastId)
t.start()
q.put(t)
q.join()
print "Messages to be downloaded: "+str(len(messList))
return messList
def getAllMessageIDForMailingList(maillist,
serverNamePort="http://localhost:5984",
threadsNumb=25):
global foundLast
# get all the date pages
dateList = getDateList(maillist)
messList = []
q = Queue.Queue()
#get the last message in couchdb
lastId = cdb.getLastEmailKey(maillist, serverNamePort)
# get all the messages relatives to the first date
t = GetIDs(maillist, dateList[0], q, messList, lastId)
t.start()
q.put(t)
q.join()
# get all the rest using threads
if not (foundLast):
for i in range(1, len(dateList), threadsNumb):
for j in range(0, threadsNumb):
if i + j >= len(dateList):
break
else:
t = GetIDs(maillist, dateList[i + j], q, messList, lastId)
t.start()
q.put(t)
q.join()
print "Messages to be downloaded: " + str(len(messList))
return messList
numMess=len(mail.list()[1])
print " Found "+str(numMess)+" messages"
#get the messages
mails = [] #empty array where messages will be stored
for i in range(numMess):
s="" #string containing a message
append=False
db="" #the maillist the mail belong to
for j in mail.retr(i+1)[1]:
s=s+j+"\n"
#check if the message belong to the mailing list
for m in maillists:
if checkLineInMaillist(j, m['name']):
db=m['name']
append=True
if(append):
mails.append(EmailParser.MapMessageMailList(db, s))
#close the connection
mail.quit()
#convert the mail as text in a nicer object
emailsObjects = EmailParser.convertTextArrayToMailArray(mails)
if(len(emailsObjects)==0):
print " No relevant mails found"
else:
print "Store in database"
CouchDBConnection.saveListOfMailCouchdb(emailsObjects, REmailS)
mail.pass_(password)
numMess = len(mail.list()[1])
print " Found " + str(numMess) + " messages"
#get the messages
mails = [] #empty array where messages will be stored
for i in range(numMess):
s = "" #string containing a message
append = False
db = "" #the maillist the mail belong to
for j in mail.retr(i + 1)[1]:
s = s + j + "\n"
#check if the message belong to the mailing list
for m in maillists:
if checkLineInMaillist(j, m['name']):
db = m['name']
append = True
if (append):
mails.append(EmailParser.MapMessageMailList(db, s))
#close the connection
mail.quit()
#convert the mail as text in a nicer object
emailsObjects = EmailParser.convertTextArrayToMailArray(mails)
if (len(emailsObjects) == 0):
print " No relevant mails found"
else:
print "Store in database"
CouchDBConnection.saveListOfMailCouchdb(emailsObjects, REmailS)
