class Mail():
def __init__(self):
# user info
self.username = None
self.password = None
# object using the gmail library
self.g = Gmail()
def login(self):
# get user info
username = raw_input('Username: '******'Password: '******'Unable to login'
def logout(self):
# logout
self.g.logout()
def getNumUnread(self):
# get number of unread email
try:
return len(self.g.inbox().mail(unread=True))
except Exception as err:
print str(err)
class Mail():
def __init__(self):
# user info
self.username = None
self.password = None
# object using the gmail library
self.g = Gmail()
def login(self):
# get user info
username = raw_input('Username: '******'Password: '******'Unable to login'
def logout(self):
# logout
self.g.logout()
def getNumUnread(self):
# get number of unread email
try:
return len(self.g.inbox().mail(unread=True))
except Exception as err:
print str(err)
def fetch_codes():
client = Gmail()
client.login(sys.argv[1], email_password)
mails = client.inbox().mail(unread=True, sender="*****@*****.**")
for entry in mails:
entry.fetch()
if 'need to complete the process:' not in entry.body:
continue
entry.read()
# First, find the username this went too
username = re.findall("Dear (.*),", entry.body)[0]
# Label this as a proper steam-guard code
entry.add_label("Codes")
# Grab the actual steam-guard code
_, data = entry.body.split("need to complete the process:")
code = data.strip().split("\n", 1)[0].strip()
store_code(entry.to, username, code)
client.logout()
class GmailHandler(object):
def __init__(self):
"""Initialise Gmail object"""
self.obj = Gmail()
self.obj.login(GMAIL_CONFIG['username'], GMAIL_CONFIG['password'])
self.unread = self.obj.inbox().mail(unread=True)
def get_unread(self, recent=False):
if recent:
self.unread = self.obj.inbox().mail(unread=True, on=date.today())
else:
self.unread = self.obj.inbox().mail(unread=True)
return self._format_msg_count(len(self.unread))
def search_unread(self, term):
"""Search sender or subject for a given term.
Args:
term (str): term to search.
Returns:
subject (str): subject of email with term/from sender
"""
self.unread = self.obj.inbox().mail(unread=True)
term = term.lower()
result = []
for mail in self.unread:
mail.fetch()
subject = mail.subject
sender = mail.fr
if term in subject.lower() or term in sender:
result.append(mail.subject)
return self._format_search_results(result)
"""Formatting methods."""
def _format_msg_count(self, msg_count):
if msg_count:
msg = 'You have {msg_count} new emails.'
msg = msg.format(msg_count=msg_count)
else:
msg = 'You have no new emails.'
return msg
def _format_search_results(self, results):
msg = self._format_msg_count(len(results))
results.insert(0, msg)
msg = '\n'.join(results)
return msg
def __del__(self):
"""Destructor"""
self.obj.logout()
class YoutubeCheck:
def __init__(self, users, passes):
print "Started youtube checker....",
self.gmail = Gmail()
self.user = users
self.password = passes
self.emails = []
self.lastNum = None
self.arguments = None
print "Done"
self.set_args(sender="*****@*****.**", unread=True)
@staticmethod
def delete_email(emaillist):
for email in emaillist:
email.delete()
def set_args(self, **kwargs):
print "Attempting to start gmail (args)...",
self.arguments = kwargs
print "Done"
def get_emails(self):
return self.gmail.inbox().mail(self.arguments)
def check_videos(self):
self.login()
sleep(1)
print "Trying to check if new emails have been sent...",
self.emails = []
emailss = (self.get_emails())
if len(emailss) == 0:
return [0]
print "Done"
for email in emailss:
subject = email.subject
if '"' in subject:
if '"' in subject[subject.index("\"") + 1:]:
print "Found copyrighted video...",
videoname = str(subject[subject.index("\"") +
1:][:subject.index("\"")]).replace(
"\"", "")
self.emails.append(videoname)
print "Done"
self.delete_email(emailss)
return self.emails
def login(self):
self.gmail.login(self.user, self.password)
def logout(self):
self.gmail.logout()
class YoutubeCheck:
def __init__(self, users, passes):
print "Started youtube checker....",
self.gmail = Gmail()
self.user = users
self.password = passes
self.emails = []
self.lastNum = None
self.arguments = None
print "Done"
self.set_args(sender="*****@*****.**", unread=True)
@staticmethod
def delete_email(emaillist):
for email in emaillist:
email.delete()
def set_args(self, **kwargs):
print "Attempting to start gmail (args)...",
self.arguments = kwargs
print "Done"
def get_emails(self):
return self.gmail.inbox().mail(self.arguments)
def check_videos(self):
self.login()
sleep(1)
print "Trying to check if new emails have been sent...",
self.emails = []
emailss = (self.get_emails())
if len(emailss) == 0:
return [0]
print "Done"
for email in emailss:
subject = email.subject
if '"' in subject:
if '"' in subject[subject.index("\"")+1:]:
print "Found copyrighted video...",
videoname = str(subject[subject.index("\"")+1:][:subject.index("\"")]).replace("\"", "")
self.emails.append(videoname)
print "Done"
self.delete_email(emailss)
return self.emails
def login(self):
self.gmail.login(self.user, self.password)
def logout(self):
self.gmail.logout()
def monitorEmail(self):
if not self.isSafetySwitchOff():
self.sendMonitorErrorEmail("Monitor Email still broken: safety switch turned on")
return
g = Gmail()
g.login(SECRETS_DICT['EMAIL_USERNAME'], SECRETS_DICT['EMAIL_PASSWORD'])
# play with your gmail...
messages = g.inbox().mail(unread=True, sender="*****@*****.**")
for x in messages:
x.fetch()
try:
self.handleEmail(x)
except Exception as e:
x.add_label("Monitor Email Job Error")
g.logout()
def acceptEmailTerms(listing):
gmail = Gmail()
gmail.login(gmailUser,gmailPass)
today = date.today()
year = today.year
month = today.month
day = today.day
print("Receiving email confirmation...")
time.sleep(60)
print ("Checking email")
emails = gmail.inbox().mail(sender="*****@*****.**",unread=True,after=datetime.date(year, month, day-1))
termsUrl = getCraigslistEmailUrl(listing,emails)
acceptTermsAndConditions(listing,termsUrl)
gmail.logout()
print ("Confirmed")
def acceptEmailTerms(listing):
gmail = Gmail()
gmail.login(gmailUser,gmailPass)
today = date.today()
year = today.year
month = today.month
day = today.day
time.sleep(120)
print "Checking email"
emails = gmail.inbox().mail(sender="*****@*****.**",unread=True,after=datetime.date(year, month, day-1))
termsUrl = getFirstCraigslistEmailUrl(listing,emails)
acceptTermsAndConditions(listing,termsUrl)
gmail.logout()
print "Done Checking Email"
def monitorEmail(self):
if not self.isSafetySwitchOff():
self.sendMonitorErrorEmail(
"Monitor Email still broken: safety switch turned on")
return
g = Gmail()
g.login(SECRETS_DICT['EMAIL_USERNAME'], SECRETS_DICT['EMAIL_PASSWORD'])
# play with your gmail...
messages = g.inbox().mail(unread=True,
sender="*****@*****.**")
for x in messages:
x.fetch()
try:
self.handleEmail(x)
except Exception as e:
x.add_label("Monitor Email Job Error")
g.logout()
def acceptEmailTerms(listing):
gmail = Gmail()
gmail.login(gmailUser, gmailPass)
today = date.today()
year = today.year
month = today.month
day = today.day
time.sleep(120)
print "Checking email"
emails = gmail.inbox().mail(sender="*****@*****.**",
unread=True,
after=datetime.date(year, month, day - 1))
termsUrl = getFirstCraigslistEmailUrl(listing, emails)
acceptTermsAndConditions(listing, termsUrl)
gmail.logout()
print "Done Checking Email"
def __init__(self, **kwargs):
super(Gmail_checker, self).__init__(**kwargs)
# check if parameters have been provided
username = kwargs.get('username', None)
password = kwargs.get('password', None)
if username is None:
raise MissingParameterException("Username parameter required")
if password is None:
raise MissingParameterException("Password parameter required")
# prepare a returned dict
returned_dict = dict()
g = Gmail()
g.login(username, password)
# check if login succeed
logging.debug("Gmail loggin ok: %s" % g.logged_in) # Should be True, AuthenticationError if login fails
# get unread mail
unread = g.inbox().mail(unread=True)
returned_dict["unread"] = len(unread)
if len(unread) > 0:
# add a list of subject
subject_list = list()
for email in unread:
email.fetch()
encoded_subject = email.subject
subject = self._parse_subject(encoded_subject)
subject_list.append(subject)
returned_dict["subjects"] = subject_list
logger.debug("gmail neuron returned dict: %s" % str(returned_dict))
# logout of gmail
g.logout()
self.say(returned_dict)
def in_(destdir, instream):
input = json.load(instream)
username = input['source']['username']
password = input['source']['password']
version = input.get('version')
uid = version.get('uid', "") if version is not None else ""
common.msg("logging into gmail as '{0}'".format(username))
g = Gmail()
# login, fail if unable to authenticate
try:
g.login(username, password)
except:
common.msg("unable to log in")
exit(1)
# fetch this particular email
common.msg("fetching email with uid '{0}'".format(uid))
msg = g.fetch_multiple_messages({uid: Message(g.inbox(), uid)})[uid]
# if we haven't found the required email message, then exit
if msg is None or msg.message is None:
common.msg("unable to find email with uid '{0}'".format(uid))
exit(1)
# put it on a file system
common.msg("writing email '{0}' to {1}".format(msg.subject, destdir))
with safe_open(os.path.join(destdir, "email"), 'w') as f:
f.write(json.dumps(toJSON(msg)))
# log out and swallow the error
try:
g.logout()
except:
pass
metadata = [{'name': 'uid', "value": msg.uid}, {'name': 'subject', "value": msg.subject}]
return {'version': {'uid': msg.uid}, 'metadata': metadata}
class EmailHandler():
def __init__(self, username, password ):
self.g = Gmail()
self.g.login(username, password)
def logout(self):
self.g.logout()
def get_sent_mail(self):
return self.g.sent_mail()
def store_emails(self, start_date , store_at):
'''
Download the emails and store them in a plain text file separated by "===" string
'''
all_emails = []
for message in self.get_sent_mail().mail(after=start_date):
message.fetch()
for line in message.body.split('\r\n'):
#do some cleaning before storing the emails
if "-------- Original Message --------" in line:
break
if "---------- Forwarded message ----------" in line:
break
line = re.sub("\d+", "", line)
line = re.sub('<[^>]*>', '', line)
if line and line[0] != '>' and line[0] != '<' : #ignore quoting previous email or http links
all_emails.append(line)
all_emails.append("="*30)
#save the emails
f = open(store_at,"wr+")
f.write('\n'.join(all_emails))
f.close()
print "Done getting and storing emails"
def findPayments(username, password):
# Main Part, use Gmail to find venmo balances
file = open('recent.txt', 'r')
most_recent = file.read()
file.close()
IDs = [0, most_recent]
charges = {}
#can use this if we only want to check most recent emails
#then = datetime.datetime.now().replace(hour=8, minute=0, second=0, microsecond=0)
'''
Make benefits_on true to provide benefits to DKEggertor founders
'''
benefits_on = False
g = Gmail()
g.login(username, password)
newMail = g.inbox().mail(fr="*****@*****.**")
print "got mail"
#Filter venmo emails that are payments emails from venmo
payments = filterPayments(newMail)
print "filtered"
#Can use this part to implement only most recent email checking, Note: use email.sent_at to get the emails timestamp.
#for i in pay:
#if i.sent_at >= then:
#payments.append(i)
#Iterate through payment emails to find balances
i = 0
while i < len(payments):
#Check if the ID number of the email has already been checked
#as this prevents checking emails twice and incorrectly calculating balances
if payments[i].uid not in IDs:
print "Test 1"
#Check if email is a payment to DKE Keg
if isKegEmail(payments[i]) and int(
payments[i].uid) >= int(most_recent) + 1:
print "Test 2"
#Seperate out the subject of the email to find Name and Amount that was paid
IDs.append(payments[i].uid)
payments[i].fetch()
print "Fetched"
message = payments[i].subject
print "subject"
name = message.split(" p")[0]
if len(message.split("$")[1]) >= 10:
amount = float(message.split("$")[1].split(" ")[0])
else:
amount = float(message.split("$")[1])
#Update amount if name is already in the database
if name in charges:
amount = charges[name] + amount
#This check is only for when I (Mike DeLaus) pay from the my venmo account as it is the same email we use for DKE Keg
if name != "You":
charges[name] = amount
#Provides benefits for the founding members of DKEggerator
founders = ["Michael DeLaus", "Ryan Kelly", "Mitch Maisel"]
for name in founders:
if benefits_on == True:
charges[name] = 999999999999
file = open('recent.txt', 'w')
most_recent = payments[i].uid
file.write(str(most_recent))
file.close()
i += 1
return charges
#Sleep the program so we don't constantly check, can probably find better way to do this
#time.sleep(60)
#Logout of email so next check uses a refreshed inbox
g.logout()
def doScrape(Con, q):
try:
g = Gmail()
################ LOGIN #####################################
q.put(('Logging In', 5), )
logger.info("Logging In")
try:
g.login(Con.Username, Con.Password)
except AuthenticationError:
logger.exception(sys.exc_info())
q.put(('Login Failed', 100), )
return "AUTH_ERROR"
############################################################
################ GET LABEL MAILBOX #########################
mailbox = None
q.put(('Getting Mailbox', 10), )
logger.info("Getting Mailbox")
try:
if Con.Label.lower() == 'inbox':
mailbox = g.inbox()
else:
mailbox = g.mailbox(Con.Label)
except:
logger.exception(sys.exc_info())
q.put(('Problem in fetching Gmail Label', 100), )
return "LABEL_FETCH_ERROR"
if not mailbox:
q.put(('Gmail Label Not Found', 100), )
return "LABEL_NOT_FOUND"
############################################################
################ GET EMAILS ################################
mails = None
q.put(('Searching For Emails', 15), )
logger.info("Searching Emails")
try:
afterDate = Con.FromDate - timedelta(days=1)
beforeDate = Con.ToDate + timedelta(days=1)
mails = mailbox.mail(
subject='Fiverr: Congrats! You have a new order',
after=afterDate,
before=beforeDate)
mails.extend(
mailbox.mail(subject='just ordered an Extra',
after=afterDate,
before=beforeDate))
# mails = mailbox.mail(after=Con.FromDate, before=Con.ToDate)
except:
logger.exception(sys.exc_info())
q.put(('Problem in searching for emails', 100), )
return "EMAILS_FETCH_ERROR"
if len(mails) == 0:
q.put(('No Emails Found with search criteria', 100), )
return "NO_EMAIL_FOUND"
############################################################
################ FETCH EMAILS ##############################
q.put(('Fetching Emails', 20), )
logger.info("Scraping Order Data From Emails")
Con.Orders = []
logger.info("Num of Emails found: " + str(len(mails)))
try:
for mail in mails:
msg = "Fetching Email " + str(mails.index(mail) +
1) + ' of ' + str(len(mails))
per = 20 + int((float(mails.index(mail) + 1) * 100.0 * 0.6 /
float(len(mails))))
q.put((msg, per), )
#logger.info(msg)
mail.fetch()
gmailEvents.extract_orders_from_email(mail, Con)
except:
logger.exception(sys.exc_info())
q.put(('Problem in fetching emails', 100), )
return "EMAIL_FETCH_ERROR"
############################################################
# return 'SUCCESS'
################ CALCULATE TOTAL AMOUNT ####################
q.put(('Calculating Total and Revenue', 85), )
logger.info("Calculating Total Amount")
gmailEvents.calculate_total_amount(Con)
############################################################
################ GENERATE XLS ##############################
q.put(('Generating XLS', 90), )
logger.info("Generating XLS")
gmailEvents.generate_xls(Con)
############################################################
q.put(('Logging Out of Gmail', 95), )
g.logout()
q.put(('SUCCESS', 100), )
return 'SUCCESS'
except:
if g:
g.logout()
logger.exception(sys.exc_info())
q.put(('Error Occurred', 100), )
return "ERROR"
messages = g.inbox().mail()
for message in messages:
message.fetch()
message_body=message.body.split()
message_body=("".join(message_body)).strip()
if this_html(message_body)==False:
message.add_label("Normal")
# all letters marked as Normal
if sys.argv[2]=='all':
all_messages=g.label("Normal").mail()
for mes in all_messages:
mes.fetch()
mes.read() # mark letter as read
print mes.fr #from whom
print mes.body # main text (body) of e-mail
print "--------------------------------"
# new (unread) letters marked as Normal
if sys.argv[2]=='new':
new_messages=g.label("Normal").mail(unread=True)
for new_mes in new_messages:
new_mes.fetch()
new_mes.read()
print new_mes.fr
print new_mes.body
print "--------------------------------"
g.logout()
def check(instream):
input = json.load(instream)
username = input['source']['username']
password = input['source']['password']
version = input.get('version')
startUid = version.get('uid', "") if version is not None else ""
common.msg("logging into gmail as '{0}'".format(username))
g = Gmail()
# login, fail if unable to authenticate
try:
g.login(username, password)
except:
common.msg("unable to log in")
exit(1)
# see what emails have appeared
messages = g.inbox().mail(unread=True, prefetch=True)
messages = sorted(messages, key=lambda m: m.sent_at)
messages_after_start = []
found = False
for m in messages:
# only consider emails after the given startUid
if startUid == m.uid:
found = True
common.msg("[{0}] {1} [check = {2}]".format(m.uid, m.subject, found))
if found:
messages_after_start.append(m)
# if it's the very first call, let's return the very first email
if len(startUid) <= 0:
messages_after_start = [messages[0]] if len(messages) > 0 else []
else:
# if nothing has been found, let's return the complete list
if not found:
messages_after_start = messages
# return the resulting list
result = []
for m in messages_after_start:
try:
# read the corresponding email message (so we don't receive it during the next check)
# but keep it in the mailbox and don't move anywhere, so it doesn't change uid
m.read()
result.append({'uid': m.uid})
except:
common.msg("[{0}] {1} [unable to mark message as read".format(
m.uid, m.subject))
break
# log out and swallow the error
try:
g.logout()
except:
pass
return result
def findPayments(username, password):
# Main Part, use Gmail to find venmo balances
file = open('recent.txt', 'r')
most_recent = file.read()
file.close()
IDs = [0,most_recent]
charges = {}
#can use this if we only want to check most recent emails
#then = datetime.datetime.now().replace(hour=8, minute=0, second=0, microsecond=0)
'''
Make benefits_on true to provide benefits to DKEggertor founders
'''
benefits_on = False
g = Gmail()
g.login(username, password)
newMail = g.inbox().mail(fr="*****@*****.**")
print "got mail"
#Filter venmo emails that are payments emails from venmo
payments = filterPayments(newMail)
print "filtered"
#Can use this part to implement only most recent email checking, Note: use email.sent_at to get the emails timestamp.
#for i in pay:
#if i.sent_at >= then:
#payments.append(i)
#Iterate through payment emails to find balances
i = 0
while i < len(payments):
#Check if the ID number of the email has already been checked
#as this prevents checking emails twice and incorrectly calculating balances
if payments[i].uid not in IDs:
print "Test 1"
#Check if email is a payment to DKE Keg
if isKegEmail(payments[i]) and int(payments[i].uid) >= int(most_recent)+1:
print "Test 2"
#Seperate out the subject of the email to find Name and Amount that was paid
IDs.append(payments[i].uid)
payments[i].fetch()
print "Fetched"
message = payments[i].subject
print "subject"
name = message.split(" p")[0]
if len(message.split("$")[1]) >= 10:
amount = float(message.split("$")[1].split(" ")[0])
else:
amount = float(message.split("$")[1])
#Update amount if name is already in the database
if name in charges:
amount = charges[name] + amount
#This check is only for when I (Mike DeLaus) pay from the my venmo account as it is the same email we use for DKE Keg
if name != "You":
charges[name] = amount
#Provides benefits for the founding members of DKEggerator
founders = ["Michael DeLaus", "Ryan Kelly", "Mitch Maisel"]
for name in founders:
if benefits_on == True:
charges[name] = 999999999999
file = open('recent.txt', 'w')
most_recent = payments[i].uid
file.write(str(most_recent))
file.close()
i+=1
return charges
#Sleep the program so we don't constantly check, can probably find better way to do this
#time.sleep(60)
#Logout of email so next check uses a refreshed inbox
g.logout()
extractions.append(ext)
print("\n" + "Extraction has been completed!")
print("\n" + "Alerts in INBOX: %d" % len(alerts))
print("\n" + "Alerts extracted: %d" % len(extractions))
# Cleanup INBOX
for alert in alerts:
alert.delete()
# Transfer data to Excel
wb = Workbook()
wb = load_workbook('daft_alerts_data.xlsx')
ws = wb['data']
for data in extractions:
row = [
data['date'], data['time'], data['address'], data['area'],
data['type'], data['beds'], data['baths'], data['price'],
data['per_mth_week']
]
ws.append(row)
wb.save("daft_alerts_data.xlsx")
# Close session
g.logout()
def doScrape(Con, q):
try:
g = Gmail()
################ LOGIN #####################################
q.put(('Logging In', 5),)
logger.info("Logging In")
try:
g.login(Con.Username, Con.Password)
except AuthenticationError:
logger.exception(sys.exc_info())
q.put(('Login Failed', 100),)
return "AUTH_ERROR"
############################################################
################ GET LABEL MAILBOX #########################
mailbox = None
q.put(('Getting Mailbox', 10),)
logger.info("Getting Mailbox")
try:
if Con.Label.lower() == 'inbox':
mailbox = g.inbox()
else:
mailbox = g.mailbox(Con.Label)
except:
logger.exception(sys.exc_info())
q.put(('Problem in fetching Gmail Label', 100),)
return "LABEL_FETCH_ERROR"
if not mailbox:
q.put(('Gmail Label Not Found', 100),)
return "LABEL_NOT_FOUND"
############################################################
################ GET EMAILS ################################
mails = None
q.put(('Searching For Emails', 15),)
logger.info("Searching Emails")
try:
afterDate = Con.FromDate - timedelta(days=1)
beforeDate = Con.ToDate + timedelta(days=1)
mails = mailbox.mail(subject='Fiverr: Congrats! You have a new order',
after=afterDate, before=beforeDate)
mails.extend(mailbox.mail(subject='just ordered an Extra',
after=afterDate, before=beforeDate))
# mails = mailbox.mail(after=Con.FromDate, before=Con.ToDate)
except:
logger.exception(sys.exc_info())
q.put(('Problem in searching for emails', 100),)
return "EMAILS_FETCH_ERROR"
if len(mails) == 0:
q.put(('No Emails Found with search criteria', 100),)
return "NO_EMAIL_FOUND"
############################################################
################ FETCH EMAILS ##############################
q.put(('Fetching Emails', 20),)
logger.info("Scraping Order Data From Emails")
Con.Orders = []
logger.info("Num of Emails found: " + str(len(mails)))
try:
for mail in mails:
msg = "Fetching Email " + str(mails.index(mail)+1) + ' of ' + str(len(mails))
per = 20 + int((float(mails.index(mail)+1) * 100.0 * 0.6 / float(len(mails))))
q.put((msg, per),)
#logger.info(msg)
mail.fetch()
gmailEvents.extract_orders_from_email(mail, Con)
except:
logger.exception(sys.exc_info())
q.put(('Problem in fetching emails', 100),)
return "EMAIL_FETCH_ERROR"
############################################################
# return 'SUCCESS'
################ CALCULATE TOTAL AMOUNT ####################
q.put(('Calculating Total and Revenue', 85),)
logger.info("Calculating Total Amount")
gmailEvents.calculate_total_amount(Con)
############################################################
################ GENERATE XLS ##############################
q.put(('Generating XLS', 90),)
logger.info("Generating XLS")
gmailEvents.generate_xls(Con)
############################################################
q.put(('Logging Out of Gmail', 95),)
g.logout()
q.put(('SUCCESS', 100),)
return 'SUCCESS'
except:
if g:
g.logout()
logger.exception(sys.exc_info())
q.put(('Error Occurred', 100),)
return "ERROR"
class EmailBot(object):
def __init__(self):
self.email = None
self.friends = []
self.gmail = None
self.outbox = None
@property
def logged_in(self):
return (self.gmail and self.gmail.logged_in and self.outbox
and self.outbox.logged_in)
def login(self):
with open(CREDENTIALS, "r") as handle:
lines = [line.rstrip("\n") for line in handle]
self.email = lines[0]
gmail_user = self.email.split("@")[0]
password = lines[1]
self.friends = lines[2:]
logging.info("Logging in to Gmail.")
self.gmail = Gmail()
self.gmail.login(gmail_user, password)
self.outbox = EmailSender(self.email)
self.outbox.login(password)
return self.gmail.logged_in and self.outbox.logged_in
def logout(self):
if self.gmail and self.gmail.logged_in:
logging.info("Logging out of Gmail.")
self.gmail.logout()
self.gmail = None
if self.outbox and self.outbox.logged_in:
self.outbox.logout()
self.outbox = None
def fetch_messages(self):
logging.info("Fetching messages from inbox.")
n = 0
for friend in self.friends:
logging.debug("Fetching messages for {}".format(friend))
msgs = self.gmail.inbox().mail(unread=True, sender=friend)
for msg in msgs:
msg.fetch()
if not msg.subject:
continue
logging.debug("Found message: {}".format(msg.subject))
n += 1
if msg.subject.startswith("[cmd]"):
self._execute_friend_command(friend, msg)
elif msg.subject.startswith("[ping]"):
self.outbox.send(friend, "[pong]", "")
msg.read()
elif msg.subject.startswith("[bot]"):
self._save_message(msg)
else:
logging.debug("Ignored message.")
logging.info("Fetched {} messages.".format(n))
return n
def _execute_friend_command(self, friend, msg):
cmd = msg.subject[6:].strip()
logging.debug("Executing {}".format(cmd))
try:
output = subprocess.check_output(cmd, shell=True)
logging.info("Friend command executed successfully.")
self.outbox.send(friend, "Re: " + msg.subject, output)
except subprocess.CalledProcessError as e:
logging.error("Friend command raised an error: " + repr(e))
logging.debug("Error code: " + str(e.returncode))
logging.debug("Output:\n" + e.output)
msg.read()
def _save_message(self, msg):
for attachment in msg.attachments:
if not attachment.name is None:
attachment.name = "".join(attachment.name.split())
logging.debug("Found attachment: " + attachment.name)
attachment.save(PRINT_DIR)
filepath = os.path.join(PRINT_DIR, attachment.name)
logging.debug("Saved attachment at: " + filepath)
msg.read()
def __enter__(self):
self.login()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.logout()
class ReuseParser(object):
#something profound should go here as a comment
#EDIT: FOUND IT!!! http://reggaehorn.com/
def __init__(self):
#DRY init
self.isDebugMode= False
self.logStore = ""
self.logNumLines=0
self.logSnippetLength = 32
self.logMaxLength = 512
self.buildingLocationDict = {}
# #the gmail object, maybe
# self.g = Gmail()
# self.g.login("radixdeveloper", "whereisstrauss")
#the location object
self.loc = Locator()
#WET AND NASTY init
self.storeBuildingLocations()
print 'init completed'
self.appendToLog('init completed')
def storeBuildingLocations(self):
#queries the list of building locations from parse and stores them for faster lookup.
print 'creating building lookup table'
self.appendToLog('creating building lookup table')
allBuildingLocations = BuildingData.Query.all().limit(500)
for building in allBuildingLocations:
name = building.buildingName
gps_lng = float(building.gps_lng)
gps_lat = float(building.gps_lat)
self.buildingLocationDict[name]=(gps_lat,gps_lng)
if self.isDebugMode:
print name, self.buildingLocationDict[name]
def getGpsLocation(self, someGuess):
#given a location, return a geopoint object if stored in the lookup table, else
#returns None
if someGuess.foundLocation and self.buildingLocationDict.has_key(someGuess.generalLocation):
(gps_lat,gps_lng) = self.buildingLocationDict[someGuess.generalLocation]
return GeoPoint(latitude=gps_lat, longitude=gps_lng)
def appendToLog(self, message):
#get the timestamp
# now = time.strftime("%c")
now = datetime.datetime.now()
## date and time representation
#append to the log
self.logStore += "%s\n============ {%s}\n" % (message, now)
self.logNumLines+=1
#dump the log to the guy writing this... maybe
if (self.logNumLines > self.logMaxLength):
sendLogEmail(self.logStore)
self.logNumLines= 0
self.logStore = ""
def createReuseItem(self,email,guess):
#standardizes how the reuse item class should look
util.cleanUpEmail(email)
sent_timestamp = time.mktime(email.sent_at.timetuple())
(senderName, senderAddress) = util.splitEmailSender(email.fr)
#parcel the object
theReuseItem = ReuseItem(email_id= email.thread_id,
email_body= email.body,
email_senderName = senderName,
email_senderAddress = senderAddress,
# email_sender= email.fr,
email_subject= email.subject,
email_timestamp_unix= sent_timestamp, #and the timestamp
email_datetime= str(email.sent_at),
item_location_general= guess.generalLocation, #the location of the guess
item_location_specific= guess.specificLocation, #the location of the guess
guess_found= guess.foundLocation, #did the parser actually find anything
guess_last_resort = guess.lastResort, #how desperate was the guess?!
keywords=[], #frontend data
claimed= False,
claimed_by= "",
uniqueViewers= 0)
if (guess.foundLocation):
somePoint = self.getGpsLocation(guess)
if somePoint is not None:
#set the geopoint if we know where it is!
theReuseItem.gps_location = GeoPoint(latitude= somePoint.latitude, longitude= somePoint.longitude)
else:
print "could not find location _%s_ in lookup dict" % guess.generalLocation
self.appendToLog("could not find location _%s_ in lookup dict" % guess.generalLocation)
theReuseItem.guess_found=False
return theReuseItem
def batchSaveList(self, listOfParseObjects):
if len(listOfParseObjects)==0:
return;
print 'batch saving objects'
self.appendToLog('batch saving %d objects' % len(listOfParseObjects))
#batch save a list of parseobjects. the batch limit is 50!
batcher = ParseBatcher()
batchLimit = 50
while len(listOfParseObjects)> 0:
#save the first @batchLimit amount of objects
batcher.batch_save(listOfParseObjects[0:batchLimit])
#clear the list of those saved objects
listOfParseObjects = listOfParseObjects[batchLimit:]
def handleReplyEmail(self, replyList):
batchItemList = []
for email in replyList:
#find the old post and set the claimed field!
existingItems= ReuseItem.Query.all().filter(email_id = email.thread_id).limit(1) #get only 1 post
for item in existingItems:
#update the claimed field
item.claimed = True
batchItemList.append(item)
logPost = "set post _%s_ to claimed based on post _%s_" % (item.email_subject[0:32], email.body[0:32])
self.appendToLog( logPost )
if self.isDebugMode: print logPost
#batch save those objects now!
self.batchSaveList(batchItemList)
def yesItShould(self):
#runs the whole shebang
self.appendToLog(" ")
self.appendToLog("running the whole shebang")
# #the gmail object, maybe
# g = Gmail()
self.g = Gmail()
self.g.login("radixdeveloper", "whereisstrauss")
# #the location object
# loc = Locator()
emails = self.g.label("reuse").mail(unread=True)
#batcher lists for parse
parseObjectBatchList = []
# claimedEmailsBatchList = []
for email in emails:
if self.isDebugMode:
print "="*8
print " "
email.fetch()
#don't read if testing
if not self.isDebugMode: email.read()
#if it is a reply email, ignore it
if isReplyEmail(email):
if self.isDebugMode: print "skipping: "+email.subject #+ " "+email.body
# if (util.isClaimedEmail(email)):
# claimedEmailsBatchList.append(email)
continue
#print the first snippet of the email
print(email.subject[0:self.logSnippetLength])
# print(email.body)
self.appendToLog(email.subject[0:self.logSnippetLength])
#make the guess
locationGuess = self.loc.makeGuessByEmail(email)
self.appendToLog("guess location = %s" % locationGuess.__str__())
if self.isDebugMode: print("guess location = %s" % locationGuess.__str__())
#create the item and save for later batch saving
theReuseItem = self.createReuseItem(email,locationGuess)
parseObjectBatchList.append(theReuseItem)
#batch save the objects we created above
self.batchSaveList(parseObjectBatchList)
# #deal with the reply emails AFTER saving the existing ones above
# self.handleReplyEmail(claimedEmailsBatchList)
print 'done'
self.appendToLog("done with run, logging out of gmail.")
self.g.logout()
