def fetchGmail():
while True:
email = None
task = None
gc.collect()
task = db.popTask()
if task is None:
time.sleep(3)
continue
if task['timestamp'] > datetime.now(pytz.UTC): # if the task is to be served in the future
db.pushTaskObject(task)
time.sleep(3)
continue
try:
email = task['email']
imap_conn = None
print 'processing', email # , 'that was queued at', task['timestamp']
state = db.getState(email, None)
print "Using fetcher", state
if state is None: continue
lastuid = int(state['lastuid'])
credentials = oauth2client.client.OAuth2Credentials.from_json(state['credentials'])
version = int(state['version'])
# renew access token if expired
if credentials.access_token_expired:
credentials.refresh(httplib2.Http())
authstring = GenerateOAuth2String(email, credentials.access_token, base64_encode=False)
imap_conn = ADV_IMAP4_SSL('imap.gmail.com')
imap_conn.authenticate('XOAUTH2', lambda x: authstring)
except KeyboardInterrupt:
# add the task again for fetching
if email: db.pushTask(email, None)
print 'interrupted'
return
except:
db.log(email=email, ip=None, module="fetcher", msg=traceback.format_exc(), level=logging.ERROR)
if imap_conn: imap_conn.logout()
continue
try:
all = getAllMailMailbox(imap_conn)
if all is None:
imap_conn.logout()
db.log(email=email, ip=None, module="fetcher", msg="all mail not enabled")
state['imap'] = True
db.storeState(email, None, state)
# add the fetching task again to the queue with 5min delay
task['timestamp'] = datetime.now(pytz.UTC) + timedelta(minutes=3)
db.pushTaskObject(task)
continue
elif 'imap' in state:
del state['imap']
db.storeState(email, None, state)
# db.markTaskForImap(email)
# db.markTaskForEmail(email)
imap_conn.select(all)
state['working'] = True
db.storeState(email, None, state)
append = False
firstTime = False
if lastuid > 0:
emails = db.getEmails(email, version, None)
append = True
else:
emails = []
firstTime = True
ok, data = imap_conn.uid('search', None, 'UID', str(lastuid + 1) + ':*')
uids = [int(d) for d in data[0].split()]
uids = uids[-LIMIT_NEMAILS:]
# ignore if the last uid is less or equal to the result
if len(uids) == 1 and lastuid >= uids[0]: uids = []
total = len(uids)
db.log(email=email, ip=None, module="fetcher", msg=str(total) + " new emails since last login")
loaded = 0
start = 0
fetchtime = 0
parsingtime = 0
while loaded < total:
tmptime = time.time()
# print str(uids[min(start, len(uids)-1)])+ ":" + str(uids[min(start+JUMP-1, len(uids)-1)])
ok, data = imap_conn.uid('fetch', str(uids[min(start, len(uids) - 1)]) + ":" + str(
uids[min(start + JUMP - 1, len(uids) - 1)]),
'(UID X-GM-LABELS FLAGS X-GM-THRID BODY.PEEK[HEADER.FIELDS (FROM TO CC Date)])')
fetchtime += (time.time() - tmptime)
# for each email
tmptime = time.time()
for i in xrange(0, len(data), 2):
loaded += 1
emails.append(data[i])
parsingtime += (time.time() - tmptime)
perc = (loaded * 100.0) / total
if len(emails) >= REFRESH_NETWORK or loaded >= total:
if append:
db.storeEmails(email, emails, version, None)
append = False
else:
# store the file
db.storeEmails(email, emails, version + 1, None)
state['version'] = version + 1
version += 1
# update state
state['lastuid'] = uids[min(start + JUMP - 1, len(uids) - 1)]
db.storeState(email, None, state)
emails = []
db.log(email=email, ip=None, module="fetcher", msg="new version %s stored in the db" % version)
start += JUMP
imap_conn.logout()
if firstTime:
db.pushNotifyDone(email)
db.log(email=email, ip=None, module="fetcher", msg="marked for email")
db.log(email=email, ip=None, module="fetcher",
msg="done fetching. Network time: %ds. Parsing time: %ds." % (fetchtime, parsingtime))
# state = db.getState(email)
if 'working' in state: del state['working']
# delete the refresh tokens for security reasons
if 'credentials' in state: del state['credentials']
db.storeState(email, None, state)
except KeyboardInterrupt:
# add the task again for fetching
if email: db.pushTask(email, None)
print 'interrupted'
return
except:
db.log(email=email, ip=None, module="fetcher", msg=traceback.format_exc(), level=logging.ERROR)
if imap_conn: imap_conn.logout()
# add the task again for fetching
if email: db.pushTask(email, None)
time.sleep(3)
continue
try:
email = task['email']
me = "em" + str(idx) + "@hobbit.media.mit.edu"
db.log(email=email, ip=None, module="notifier", msg="sending email")
if 'done' in task:
sendEmailDone(me, email)
db.log(email=email,
ip=None,
module="notifier",
msg="sent done email from em" + str(idx))
elif 'imap' in task:
sendEmailImap(me, email)
db.log(email=email,
ip=None,
module="notifier",
msg="sent imap email from em" + str(idx))
idx += 1
if idx == 10: idx = 1
except:
# add the email task again to the queue
if 'done' in task: db.pushNotifyDone(email)
elif 'imap' in task: db.pushNotifyImap(email)
db.log(email=email,
ip=None,
module="notifier",
msg=traceback.format_exc(),
level=logging.ERROR)