def handle_timeout(self):
"""
Force remove a job from jobs_active when no finished was received for a
given timout interval.
"""
timeout_after = 300 #in seconds
time_now = time.time()
to_delete = []
for job_tuple, job_info in self.jobs_active.items():
dispatch = job_info["dispatch"]
time_since_dispatch = dispatch - time_now
if time_since_dispatch > timeout_after:
to_delete.append(job_tuple)
for job_tuple in to_delete:
del self.jobs_active[job_tuple]
logmsg = (
"Dropped a submission for User {0}, TaskNr {1} as the tester "
"did not return a finished message after 5 minutes. This "
"submission email will not be moved to the archive!")
c.log_a_msg(self.queues["logger"], self.name, logmsg, "WARNING")
def check_and_init_db_table(dbname, tablename, fields):
"""
Check if a table exists, if not create it with fields.
"""
cur, con = c.connect_to_db(dbname, logger_queue, "autosub.py")
data = {'name': tablename}
sql_cmd = "SELECT name FROM sqlite_master WHERE type == 'table' AND name = :name"
cur.execute(sql_cmd, data)
res = cur.fetchall()
if res:
logmsg = 'table ' + tablename + ' exists'
c.log_a_msg(logger_queue, "autosub.py", logmsg, "DEBUG")
#TODO: in this case, we might want to check if one entry per task is already
# there, and add new empty entries in case a task does not have one. This is only
# a problem, if the number of tasks in the config file is changed AFTER the
# TaskStats table has been changed!
con.close()
return 0
else:
logmsg = 'table ' + tablename + ' does not exist'
c.log_a_msg(logger_queue, "autosub.py", logmsg, "DEBUG")
data = {'fields': fields}
sql_cmd = "CREATE TABLE {0}({1})".format(tablename, fields)
cur.execute(sql_cmd)
con.commit()
con.close()
return 1
def a_question_was_asked(self, user_email, mail, messageid):
""""
Process a question that was asked by a user.
"""
mail_subject = str(mail['subject'])
logmsg = 'The user has a question, please take care of that!'
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
c.send_email(self.sender_queue, user_email, "", "Question", "", "", "")
# was the question asked to a specific task_nr that is valid?
search_obj = re.search('[0-9]+', mail_subject, )
if (search_obj != None) and int(search_obj.group()) <= c.get_num_tasks(self.coursedb, \
self.logger_queue, self.name):
fwd_mails = self.get_taskoperator_emails(search_obj.group())
else:
fwd_mails = self.get_admin_emails()
for mail_address in fwd_mails:
c.send_email(self.sender_queue, mail_address, "", "QFwd", "", mail, messageid)
c.increment_db_statcounter(self.semesterdb, 'nr_questions_received', \
self.logger_queue, self.name)
def get_registration_deadline(self):
"""
Get the registration deadline datetime.
Return datetime, if not found return 1 hour from now.
"""
curc, conc = c.connect_to_db(self.dbs["course"], self.queues["logger"],
self.name)
sql_cmd = ("SELECT Content FROM GeneralConfig " \
"WHERE ConfigItem == 'registration_deadline'")
curc.execute(sql_cmd)
deadline_string = str(curc.fetchone()[0])
conc.close()
format_string = '%Y-%m-%d %H:%M:%S'
if deadline_string != 'NULL':
return datetime.datetime.strptime(deadline_string, format_string)
else:
# there is no deadline set, just assume it is in 1h from now.
date = datetime.datetime.now() + datetime.timedelta(0, 3600)
logmsg = "No Registration Deadline found, assuming: " + str(date)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return date
def run(self):
logmsg = "Starting " + self.name
c.log_a_msg(self.queues["logger"], self.name, logmsg, "INFO")
while True:
self.activator_loop()
time.sleep(1800) # it's more than enough to check every half hour
def get_scriptpath(self, task_nr):
"""
Get the path to the tester script for task task_nr.
Returns None if not proper config.
"""
curc, conc = c.connect_to_db(self.dbs["course"], self.queues["logger"], self.name)
data = {'task_nr': task_nr}
sql_cmd = ("SELECT TaskName, TestExecutable FROM TaskConfiguration "
"WHERE TaskNr == :task_nr")
curc.execute(sql_cmd, data)
res = curc.fetchone()
if not res:
logmsg = ("Failed to fetch Configuration for TaskNr: {0} from the "
"database! Table TaskConfiguration corrupted?").format(task_nr)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
scriptpath = None
else:
task_name = res[0]
tester_name = res[1]
scriptpath = self.tasks_dir + "/" + task_name + "/" + tester_name
conc.close()
return scriptpath
def assemble_email(self, msg, message_text, attachments):
"""
assemble e-mail content
"""
msg.attach(MIMEText(message_text, 'plain', 'utf-8'))
# If the message is a task description, we might want to
# add some attachments. These ar given as a list by the
# attachments parameter
logmsg = "List of attachements: {0}".format(attachments)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
if str(attachments) != 'None':
for next_attachment in attachments:
try:
part = MIMEBase('application', "octet-stream")
part.set_payload(open(next_attachment, "rb").read())
encoders.encode_base64(part)
part.add_header('Content-Disposition', 'attachment; filename="{0}"'.format(os.path.basename(next_attachment)))
msg.attach(part)
except:
logmsg = "Faild to add an attachement: {0}".format(next_attachment)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
# The following message my be helpful during debugging - but
# if you use attachments, your log-file will grow very fast
# therefore it was commented out.
# logmsg = "Prepared message: \n" + str(msg)
# c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
return msg
def assemble_email(self, msg, message_text, attachments):
"""
assemble e-mail content
"""
msg.attach(MIMEText(message_text, 'plain', 'utf-8'))
# If the message is a task description, we might want to
# add some attachments. These ar given as a list by the
# attachments parameter
logmsg = "List of attachements: {0}".format(attachments)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
if str(attachments) != 'None':
for next_attachment in attachments:
try:
part = MIMEBase('application', "octet-stream")
part.set_payload(open(next_attachment, "rb").read())
encoders.encode_base64(part)
part.add_header(
'Content-Disposition',
'attachment; filename="{0}"'.format(
os.path.basename(next_attachment)))
msg.attach(part)
except:
logmsg = "Faild to add an attachement: {0}".format(
next_attachment)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
# The following message my be helpful during debugging - but
# if you use attachments, your log-file will grow very fast
# therefore it was commented out.
# logmsg = "Prepared message: \n" + str(msg)
# c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
return msg
def check_and_init_db_table(dbname, tablename, fields):
"""
Check if a table exists, if not create it with fields.
"""
cur, con = c.connect_to_db(dbname, logger_queue, "autosub.py")
data = {'name': tablename}
sql_cmd = ("SELECT name FROM sqlite_master "
"WHERE type == 'table' AND name = :name")
cur.execute(sql_cmd, data)
res = cur.fetchall()
if res:
logmsg = 'table ' + tablename + ' exists'
c.log_a_msg(logger_queue, "autosub.py", logmsg, "DEBUG")
return 0
logmsg = 'table ' + tablename + ' does not exist'
c.log_a_msg(logger_queue, "autosub.py", logmsg, "DEBUG")
data = {'fields': fields}
sql_cmd = "CREATE TABLE {0}({1})".format(tablename, fields)
cur.execute(sql_cmd)
con.commit()
con.close()
return 1
def read_specialmessage(self, msgname):
"""
read a special message from the DB
"""
curc, conc = c.connect_to_db(self.dbs["course"], \
self.queues["logger"], \
self.name)
data = {'msgname': msgname}
sql_cmd = ("SELECT EventText FROM SpecialMessages "
"WHERE EventName = :msgname")
curc.execute(sql_cmd, data)
res = curc.fetchone()
conc.close()
if not res:
logmsg = ("Error reading the Specialmessage named {0} from the "
"database").format(msgname)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
message = "Hi something went wrong. Please contact the course admin"
else:
message = str(res[0])
return message
def run(self):
logmsg = "Starting " + self.name
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
while True:
self.activator_loop()
time.sleep(3600) # it's more than enough to check every hour!
def backup_message(self, messageid):
"""
trigger archivation of an e-mail
"""
logmsg = "request backup of message with messageid: {0}".format(messageid)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
self.arch_queue.put(dict({"mid": messageid}))
def backup_message(self, messageid):
"""
trigger archivation of an e-mail
"""
logmsg = "request backup of message with messageid: {0}".format(
messageid)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
self.arch_queue.put(dict({"mid": messageid}))
def archive_message(self, message_id, is_finished_job=False):
"""
trigger archivation of an e-mail
"""
logmsg = "request archiving of message with message_id {0}".format(message_id)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG")
c.archive_message(self.queues['archive'], message_id, is_finished_job)
def run(self):
"""
Thread code for the generator thread.
"""
c.log_a_msg(self.logger_queue, self.name, "Task Generator thread started", "INFO")
while True:
self.generator_loop()
def run(self):
"""
Thread code for the generator thread.
"""
c.log_a_msg(self.logger_queue, self.name,
"Task Generator thread started", "INFO")
while True:
self.generator_loop()
def run(self):
"""
the thread code is just a tight loop that waits on the sender_queue
for some work.
"""
c.log_a_msg(self.logger_queue, self.name, \
"Starting Mail Sender Thread!", "INFO")
while True:
self.handle_next_mail()
def run(self):
"""
the thread code is just a tight loop that waits on the sender_queue
for some work.
"""
c.log_a_msg(self.queues["logger"], self.name, \
"Starting Mail Sender Thread!", "INFO")
while True:
self.handle_next_mail()
def check_last_log_entry(self, logmsg, loglvl):
common.log_a_msg(self.logger_queue, "testlogger", logmsg, loglvl)
time.sleep(.25) # give the logger some time...
with open(self.lfile, "r") as fd_log:
for line in fd_log.readlines():
#nothing - only interested in the very last line!
pass
self.assertNotEqual(line.find(logmsg), str(-1))
self.assertNotEqual(line.find(loglvl), str(-1))
def check_and_create_table(self, cur, tablename):
data = {'Table': tablename}
sql_cmd = "SELECT name FROM sqlite_master WHERE type == 'table' AND name == :Table;"
cur.execute(sql_cmd, data)
res = cur.fetchall()
if res:
logmsg = "table {0} exists".format(tablename)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
else:
logmsg = "table {0} does not exist ... creating it now".format(tablename)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
sql_cmd = "CREATE TABLE {0} (TimeStamp STRING PRIMARY KEY, value INT)".format(tablename)
cur.execute(sql_cmd)
def idmap_new_emails(self, m):
"""
Search for new (unseen) e-mails from the Inbox.
Return a mapping as dict that assigns each Message-Id its UID.
Note: This is done with read rights. All unread flags are unset.
"""
# Debug Log Message
logmsg = ("Start looking for unseen mails")
c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG")
try:
m.select("Inbox", readonly=False)
except Exception as e:
logmsg = "Failed to select inbox with error: " + str(e)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return {}
idmap = dict()
try:
resp, data = m.uid('search', None, "UNSEEN")
except Exception as e:
logmsg = "Failed to get messages from inbox with IMAP error: " + str(
e)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return {}
if resp == 'OK':
for uid in data[0].split():
try:
typ, msg_data = m.uid('fetch', uid, "(BODY[HEADER])")
except Exception as e:
logmsg = "Failed to fetch from inbox with error {0}.".format(
str(e))
c.log_a_msg(self.queues["logger"], self.name, logmsg,
"ERROR")
return {}
mail = email.message_from_bytes(msg_data[0][1])
idmap[mail['Message-ID']] = uid
return idmap
else:
logmsg = (
"Failed to get messages from inbox. Got response from IMAP "
"server: {0} ").format(resp)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return {}
def action_by_subject(self, user_id, user_email, messageid, mail,
mail_subject):
if re.search('[Rr][Ee][Ss][Uu][Ll][Tt]', mail_subject):
###############
# RESULT #
###############
searchObj = re.search('[0-9]+', mail_subject)
if searchObj == None:
# Result + no number
logmsg = ("Got a kind of message I do not understand. "
"Sending a usage mail...")
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
c.send_email(self.sender_queue, user_email, "", "Usage", "", \
"", messageid)
return
#Result + number
task_nr = searchObj.group()
self.a_result_was_submitted(user_id, user_email, task_nr, messageid, \
mail)
elif re.search('[Qq][Uu][Ee][Ss][Tt][Ii][Oo][Nn]', mail_subject):
###############
# QUESTION #
###############
self.a_question_was_asked(user_id, user_email, mail, messageid)
elif re.search('[Ss][Tt][Aa][Tt][Uu][Ss]', mail_subject):
###############
# STATUS #
###############
self.a_status_is_requested(user_id, user_email, messageid)
elif (self.allow_skipping == True) and re.search(
'[Ss][Kk][Ii][Pp]', mail_subject):
####################
# SKIP, IF ALLOWED #
####################
self.skip_was_requested(user_id, user_email, messageid)
else:
#####################
# DEFAULT ACTION #
#####################
logmsg = ("Got a kind of message I do not understand. "
"Sending a usage mail...")
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
c.send_email(self.sender_queue, user_email, "", "Usage", "", \
"", messageid)
def run(self):
"""
Thread code for the fetcher thread.
"""
c.log_a_msg(self.queues["logger"], self.name,
"Starting Mail Fetcher Thread!", "INFO")
logmsg = "Imapserver: '" + self.imap_info["server"] + "'"
c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG")
# Setting server timeout for mail server
socket.setdefaulttimeout(self.imap_info["timeout"])
logmsg = "Setting server timeout for mail server to {0} seconds".format(
self.imap_info["timeout"])
c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG")
# This thread is running as a daemon thread, this is the while(1) loop that is
# running until the thread is stopped by the main thread
while True:
self.loop_code()
logmsg = "Exiting fetcher - this should NEVER happen!"
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
def read_text_file(self, path_to_msg):
"""
read a text file
"""
try:
fpin = open(path_to_msg, 'r')
message_text = fpin.read()
fpin.close()
except:
message_text = "Even the static file was not available!"
c.log_a_msg(self.logger_queue, self.name, \
"Failed to read from config file", "WARNING")
return message_text
def read_text_file(self, path_to_msg):
"""
read a text file
"""
try:
fpin = open(path_to_msg, 'r')
message_text = fpin.read()
fpin.close()
except:
message_text = "Even the static file was not available!"
c.log_a_msg(self.logger_queue, self.name, \
"Failed to read from config file", "WARNING")
return message_text
def connect_to_imapserver(self):
"""
Connect to configured IMAP server.
"""
try:
# connecting to the imap server
m = imaplib.IMAP4_SSL(self.imapserver)
m.login(self.autosub_user, self.autosub_pwd)
except imaplib.IMAP4.abort:
logmsg = ("Login to server was aborted (probably a server-side problem). "
"Trying to connect again ...")
c.log_a_msg(self.logger_queue, self.name, logmsg, "ERROR")
#m.close()
return 0
except imaplib.IMAP4.error:
logmsg = ("Got an error when trying to connect to the imap server. "
"Trying to connect again ...")
c.log_a_msg(self.logger_queue, self.name, logmsg, "ERROR")
return 0
except:
logmsg = ("Got an unknown exception when trying to connect to the imap "\
"server. Trying to connect again ...")
c.log_a_msg(self.logger_queue, self.name, logmsg, "ERROR")
return 0
logmsg = "Successfully logged into imap server"
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
return m
def handle_job(self, nextjob):
"""
Run the test script for the given submission and act on the test result
"""
task_nr = str(nextjob.get('task_nr'))
user_id = str(nextjob.get('user_id'))
user_email = str(nextjob.get('user_email'))
message_id = str(nextjob.get('message_id'))
logmsg = "{0} got a new job: {1} from the user with id {2}".format(self.name, task_nr, \
user_id)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "INFO")
# get the task parameters
task_params = self.get_task_parameters(user_id, task_nr)
# get the path to the test script
scriptpath = self.get_scriptpath(task_nr)
# get the configured common file
configured_backend_interface = self.get_configured_backend_interface(
task_nr)
if not scriptpath:
logmsg = "Could not fetch test script from database"
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return
if not os.path.isfile(scriptpath):
logmsg = "Test script does not exist"
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return
# run the test script and log the stderr and stdout
command = [
scriptpath, user_id, task_nr, task_params,
configured_backend_interface
]
logmsg = "Running test script with arguments: {0}".format(command)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG")
# Popen in asynch, but communicate waits
process = Popen(command, stdout=PIPE, stderr=PIPE)
test_msg, test_error = process.communicate()
test_msg = test_msg.decode('UTF-8')
test_error = test_error.decode('UTF-8')
test_res = process.returncode
log_src = "Tester{0}({1})".format(str(task_nr), user_id)
if test_msg:
c.log_task_msg(self.queues["logger"], log_src, test_msg, "INFO")
if test_error:
c.log_task_error(self.queues["logger"], log_src, test_error,
"ERROR")
# act based on the result
self.handle_test_result(test_res, user_id, user_email, task_nr,
message_id)
def fetch_all_emails(self, m):
"""
Fetch all emails.
Return list of mailids.
"""
try:
m.select(mailbox='Inbox', readonly=False)
except:
logmsg = "Failed to select inbox"
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
resp, items = m.search(None, 'All')
return items[0].split()
def run(self):
"""
Thread code for the worker thread.
"""
logmsg = "Starting " + self.name
c.log_a_msg(self.queues["logger"], self.name, logmsg, "INFO")
while True:
logmsg = self.name + ": waiting for a new job."
c.log_a_msg(self.queues["logger"], self.name, logmsg, "INFO")
# blocking wait on job queue
nextjob = self.queues["job"].get(True)
self.handle_job(nextjob)
def fetch_all_emails(self, m):
"""
Fetch all emails.
Return list of mailids.
"""
try:
m.select(mailbox='Inbox', readonly=False)
except:
logmsg = "Failed to select inbox"
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
return []
resp, items = m.search(None, 'All')
return items[0].split()
def check_and_set_last_done(self, curs, cons, userid):
"""
The LastDone flag is used to mark users who have successfully solved
all tasks.
"""
data = {'uid': userid}
sql_cmd = "SELECT LastDone FROM Users WHERE UserId == :uid;"
curs.execute(sql_cmd, data)
res = curs.fetchone()
logmsg = "RES: "+ str(res[0])
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
if str(res[0]) == 'None':
data = {'uid': str(userid), 'now': str(int(time.time()))}
sql_cmd = "UPDATE Users SET LastDone = datetime(:now, 'unixepoch', 'localtime') WHERE UserId == :uid;"
curs.execute(sql_cmd, data)
cons.commit()
def disconnect_from_imapserver(self, m):
"""
Disconnect from existing imap connection
"""
# m==0 is only possible in test-code (e.g. load_test.py)
if m != 0:
try:
m.close()
m.logout()
logmsg = "closed connection to imap server"
c.log_a_msg(self.queues["logger"], self.name, logmsg, "INFO")
except Exception as e:
logmsg = (
"Got an error when trying to disconnect from the imap "
"server: {0}").format(str(e))
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
def send_out_email(self, recipient, message, msg_type):
"""
connect to the smtp server and send out an e-mail
"""
try:
# port 465 doesn't seem to work!
server = smtplib.SMTP(self.smtpserver, 587)
server.ehlo()
server.starttls()
server.login(self.autosub_user, self.mail_pwd)
server.sendmail(self.mail_user, recipient, message)
server.close()
c.log_a_msg(self.logger_queue, self.name, "Successfully sent an e-mail of type '{0}'!".format(msg_type), "DEBUG")
c.increment_db_statcounter(self.semesterdb, 'nr_mails_sent', \
self.logger_queue, self.name)
except:
c.log_a_msg(self.logger_queue, self.name, "Failed to send out an e-mail of type '{0}'!".format(msg_type), "ERROR")
def fetch_new_emails(self, m):
"""
Fetch new (unseen e-mails from the Inbox.
Return list of mailids
"""
try:
m.select("Inbox") # here you a can choose a mail box like INBOX instead
# use m.list() to get all the mailboxes
except:
logmsg = "Failed to select inbox"
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
# you could filter using the IMAP rules here
# (check http://www.example-code.com/csharp/imap-search-critera.asp)
resp, items = m.search(None, "UNSEEN")
return items[0].split()
def loop_code(self):
"""
The code run in the while True loop of the mail fetcher thread.
"""
m = self.connect_to_imapserver()
if m != 0:
self.archive_processed(m)
self.handle_timeout()
self.handle_backlogged(m)
self.handle_new(m)
self.disconnect_from_imapserver(m)
# Debug Log Message
logmsg = ("Start new sleep period")
c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG")
time.sleep(self.poll_period)
def clean_taskconfig(self):
conc = lite.connect(self.coursedb)
curc = conc.cursor()
#Drop the task configuration table (start clean)
try:
sqlcmd = "DROP table TaskConfiguration"
curc.execute(sqlcmd)
conc.commit()
except:
logmsg = "No Table TaskConfiguration?"
c.log_a_msg(self.logger_queue, "Debugger", logmsg, "DEBUG")
sqlcmd = ("CREATE TABLE TaskConfiguration (TaskNr INT PRIMARY KEY, "
"TaskStart DATETIME, TaskDeadline DATETIME, PathToTask TEXT, "
"GeneratorExecutable TEXT, TestExecutable TEXT, Score INT, "
"TaskOperator TEXT, TaskActive BOOLEAN);")
curc.execute(sqlcmd)
conc.commit()
conc.close()
def fetch_new_emails(self, m):
"""
Fetch new (unseen e-mails from the Inbox.
Return list of mailids
"""
try:
m.select(
"Inbox") # here you a can choose a mail box like INBOX instead
# use m.list() to get all the mailboxes
except:
logmsg = "Failed to select inbox"
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
return []
# you could filter using the IMAP rules here
# (check http://www.example-code.com/csharp/imap-search-critera.asp)
resp, items = m.search(None, "UNSEEN")
return items[0].split()
def connect_to_imapserver(self):
"""
Connect to configured IMAP server.
"""
try:
# connecting to the imap server
if self.imap_info["security"] == 'ssl':
server = imaplib.IMAP4_SSL(self.imap_info["server"],
int(self.imap_info["port"]))
else:
server = imaplib.IMAP4(self.imap_info["server"],
int(self.imap_info["port"]))
if self.imap_info["security"] == 'starttls':
server.starttls()
server.login(self.imap_info["user"], self.imap_info["passwd"])
except imaplib.IMAP4.abort:
logmsg = "Login to server was aborted with security= " + self.imap_info["security"] + \
" , port= " + str(self.imap_info["port"])
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return 0
except imaplib.IMAP4.error:
logmsg = "Got an error when trying to connect to the imap server with" + \
" security= " + self.imap_info["security"] + " , port= " + str(self.imap_info["port"])
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return 0
except Exception as e:
logmsg = str(e)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
logmsg = "Got an unknown exception when trying to connect to the imap " + \
"server with security= " + self.imap_info["security"] + " , port= " + str(self.imap_info["port"])
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return 0
logmsg = "Successfully logged into imap server with security= " + self.imap_info["security"] + \
" , port= " + str(self.imap_info["port"])
c.log_a_msg(self.queues["logger"], self.name, logmsg, "INFO")
return server
def a_task_is_requested(self, user_id, user_email, task_nr, message_id):
"""
Process a request for a certain task_nr. Check if that task
exists, if it is active, and if the deadline has not passed
yet. If yes put in generator queue.
"""
logmsg = "Processing a Task Request, UserId:{0} TaskNr:{1}"\
.format(user_id, task_nr)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "INFO")
#task with this task_nr exists?
is_task = c.is_valid_task_nr(self.dbs["course"], task_nr, self.queues["logger"], \
self.name)
if not is_task:
# task_nr is not valid
c.send_email(self.queues["sender"], user_email, "", "InvalidTask", str(task_nr), \
"", message_id)
return
# get time now, deadline and starttime of task
time_now = datetime.datetime.now()
starttime = c.get_task_starttime(self.dbs["course"], task_nr, \
self.queues["logger"], self.name)
deadline = c.get_task_deadline(self.dbs["course"], task_nr, \
self.queues["logger"], self.name)
if not (starttime <= time_now):
# task not active yet
logmsg = ("Task not active")
c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG")
c.send_email(self.queues["sender"], user_email, "", "TaskNotActive", \
str(task_nr), "", message_id)
return
if deadline < time_now:
# deadline passed for that task_nr!
logmsg = ("Deadline passed")
c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG")
c.send_email(self.queues["sender"], user_email, "", "DeadTask", \
str(task_nr), "", message_id)
return
logmsg = ("Calling Generator to create"
"TaskNr:{0} for UserId:{1}").format(task_nr, user_id)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG")
c.generate_task(self.queues["generator"], user_id, task_nr, user_email, \
message_id)
def idmap_all_emails(self, m):
"""
Search for all emails.
Return a mapping as dict that assigns each Message-Id its UID.
Note: This is done readonly to preserve unread flags.
"""
try:
m.select(mailbox='Inbox', readonly=True)
except Exception as e:
logmsg = "Failed to select inbox with error " + str(e)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return {}
idmap = dict()
try:
resp, items = m.uid('search', None, "ALL")
except Exception as e:
logmsg = "Failed to get messages from inbox with IMAP error: " + str(
e)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return {}
if resp == 'OK':
for uid in items[0].split():
try:
typ, msg_data = m.uid('fetch', uid, "(BODY[HEADER])")
except Exception as e:
logmsg = ("Failed to fetch message with uid {0} from "
"inbox with error {1} ").format(uid, str(e))
c.log_a_msg(self.queues["logger"], self.name, logmsg,
"ERROR")
return {}
mail = email.message_from_bytes(msg_data[0][1])
idmap[mail['Message-ID']] = uid
return idmap
else:
logmsg = (
"Failed to get messages from inbox. Got response from IMAP "
"server: {0} ").format(resp)
c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR")
return {}
def a_status_is_requested(self, user_id, user_email, messageid):
"""
Tell sender to send out a status email.
"""
logmsg = ("STATUS requested: User with UserId:{0}, Email: {1}").format(\
user_id, user_email)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
curs, cons = c.connect_to_db(self.semesterdb, self.logger_queue,
self.name)
data = {'user_id': user_id}
sql_cmd = "SELECT CurrentTask FROM Users WHERE UserId == :user_id"
curs.execute(sql_cmd, data)
res = curs.fetchone()
current_task = res[0]
cons.close()
c.send_email(self.sender_queue, user_email, user_id, "Status", current_task, \
"", messageid)
c.increment_db_statcounter(self.semesterdb, 'nr_status_requests', \
self.logger_queue, self.name)
def check_if_whitelisted(self, user_email):
"""
Check if the given e-mail address is in the whitelist.
"""
curs, cons = c.connect_to_db(self.semesterdb, self.logger_queue, self.name)
data = {'Email': user_email}
sql_cmd = "SELECT * FROM WhiteList WHERE Email == :Email"
curs.execute(sql_cmd, data)
res = curs.fetchone()
cons.close()
if res != None:
return 1
else:
logmsg = "Got Mail from a User not on the WhiteList: " + user_email
c.log_a_msg(self.logger_queue, self.name, logmsg, "Warning")
c.increment_db_statcounter(self.semesterdb, 'nr_non_registered', \
self.logger_queue, self.name)
return 0
def check_if_whitelisted(self, user_email):
"""
Check if the given e-mail address is in the whitelist.
"""
curs, cons = c.connect_to_db(self.dbs["semester"],
self.queues["logger"], self.name)
data = {'Email': user_email}
sql_cmd = "SELECT * FROM WhiteList WHERE Email == :Email"
curs.execute(sql_cmd, data)
res = curs.fetchone()
cons.close()
if res != None:
return True
logmsg = "Got mail from a user not on the WhiteList: " + user_email
c.log_a_msg(self.queues["logger"], self.name, logmsg, "Warning")
c.increment_db_statcounter(self.dbs["semester"], 'nr_non_registered', \
self.queues["logger"], self.name)
return False
def add_new_user(self, user_name, user_email):
"""
Add the necessary entries to database for a newly registered user.
"""
curs, cons = c.connect_to_db(self.semesterdb, self.logger_queue, self.name)
logmsg = 'New Account: User: %s' % user_name
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
data = {'Name': user_name, 'Email': user_email, 'TimeNow': str(int(time.time()))}
sql_cmd = ("INSERT INTO Users "
"(UserId, Name, Email, FirstMail, LastDone, CurrentTask) "
"VALUES(NULL, :Name, :Email, datetime(:TimeNow, 'unixepoch', 'localtime')"
", NULL, 1)")
curs.execute(sql_cmd, data)
cons.commit()
# the new user has now been added to the database. Next we need
# to send him an email with the first task.
# read back the new users UserId and create a directory for putting his
# submissions in.
data = {'Email': user_email}
sql_cmd = "SELECT UserId FROM Users WHERE Email = :Email"
curs.execute(sql_cmd, data)
res = curs.fetchone()
user_id = str(res[0])
dir_name = 'users/'+ user_id
c.check_dir_mkdir(dir_name, self.logger_queue, self.name)
cons.close()
# NOTE: messageid is empty, cause this will be sent out by the welcome message!
curc, conc = c.connect_to_db(self.coursedb, self.logger_queue, self.name)
sql_cmd = "SELECT GeneratorExecutable FROM TaskConfiguration WHERE TaskNr == 1"
curc.execute(sql_cmd)
res = curc.fetchone()
conc.close()
if res != None:
logmsg = "Calling Generator Script: " + str(res[0])
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
logmsg = "UserID " + user_id + ",UserEmail " + user_email
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
self.gen_queue.put(dict({"user_id": user_id, "user_email": user_email, \
"task_nr": "1", "message_id": ""}))
else:
# If there is no generator script, we assume, that there is a static
# description.txt which shall be used.
c.send_email(self.sender_queue, user_email, user_id, "Task", "1", "", "")
def run(self):
"""
Thread code for the fetcher thread.
"""
c.log_a_msg(self.logger_queue, self.name, "Starting Mail Fetcher Thread!", "INFO")
logmsg = "Imapserver: '" + self.imapserver + "'"
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
# This thread is running as a daemon thread, this is the while(1) loop that is
# running until the thread is stopped by the main thread
while True:
self.loop_code()
logmsg = "Exiting fetcher - this should NEVER happen!"
c.log_a_msg(self.logger_queue, self.name, logmsg, "ERROR")
def run(self):
"""
Thread code for the worker thread.
"""
logmsg = "Starting " + self.name
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
while True:
logmsg = self.name + ": waiting for a new job."
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
nextjob = self.job_queue.get(True)
if nextjob:
tasknr = nextjob.get("taskNr")
user_id = nextjob.get("UserId")
user_email = nextjob.get("UserEmail")
message_id = nextjob.get("MessageId")
logmsg = self.name + "got a new job: {0} from the user with id: {1}".format(str(tasknr), str(user_id))
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
# check if there is a test executable configured in the
# database -- if not fall back on static test script.
curc, conc = c.connect_to_db(self.coursedb, self.logger_queue, self.name)
try:
data = {"tasknr": tasknr}
sql_cmd = "SELECT TestExecutable FROM TaskConfiguration WHERE TaskNr == :tasknr"
curc.execute(sql_cmd, data)
testname = curc.fetchone()
except:
logmsg = "Failed to fetch TestExecutable for TaskNr: {0}".format(tasknr)
logmsg = logmsg + " from the Database! Table TaskConfiguration corrupted?"
c.log_a_msg(self.logger_queue, self.name, logmsg, "ERROR")
if testname != None:
try:
data = {"tasknr": tasknr}
sql_cmd = "SELECT PathToTask FROM TaskConfiguration WHERE TaskNr == :tasknr"
curc.execute(sql_cmd, data)
path = curc.fetchone()
scriptpath = str(path[0]) + "/" + str(testname[0])
except: # if a testname was given, then a Path should be
# there as well!
logmsg = "Failed to fetch Path to Tasknr: {0}".format(tasknr)
logmsg = "{0} from the Database! Table TaskConfiguration corrupted?".format(logmsg)
c.log_a_msg(self.logger_queue, self.name, logmsg, "ERROR")
else: # in case no testname was given, we fall back to the
# static directory structure
scriptpath = "tasks/task" + str(tasknr) + "/./tests.sh"
conc.close()
# get the task parameters
task_params = self.get_task_parameters(user_id, tasknr)
# run the test script
logmsg = "Running test script: " + scriptpath
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
command = '{0} {1} {2} "{3}" >> autosub.stdout 2>>autosub.stderr'.format(
scriptpath, user_id, tasknr, task_params
)
test_res = os.system(command)
if test_res: # not 0 returned
logmsg = "Test failed! User: {0} Task: {1}".format(user_id, tasknr)
logmsg = logmsg + " return value:" + str(test_res)
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
c.send_email(
self.sender_queue, str(user_email), str(user_id), "Failed", str(tasknr), "", str(message_id)
)
if test_res == 512: # Need to read up on this but os.system() returns
# 256 when the script returns 1 and 512 when the script returns 2, 768 when 3!
logmsg = "SecAlert: This test failed due to probable attack by user!"
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
c.send_email(
self.sender_queue,
str(user_email),
str(user_id),
"SecAlert",
str(tasknr),
"",
str(message_id),
)
elif test_res == 768:
logmsg = "TaskAlert: This test for TaskNr {0} and User {1} failed due an error with task/testbench analyzation!".format(
tasknr, user_id
)
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
c.send_email(
self.sender_queue,
str(user_email),
str(user_id),
"TaskAlert",
str(tasknr),
"",
str(message_id),
)
else: # 0 returned
logmsg = "Test succeeded! User: {0} Task: {1}".format(user_id, tasknr)
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
# Notify, the user that the submission was successful
c.send_email(self.sender_queue, str(user_email), str(user_id), "Success", str(tasknr), "", "")
curc, conc = c.connect_to_db(self.coursedb, self.logger_queue, self.name)
currenttask = int(c.user_get_current_task(self.semesterdb, user_id, self.logger_queue, self.name))
# Next, a new Task is generated -- but only if a new task
# exists AND if a generator script exists (otherwise
# static task description is assumed, AND if users current
# task < the task that shall be generated (no Task has yet
# been generated for this user yet).
if currenttask < int(tasknr) + 1:
try:
data = {"tasknr": str(int(tasknr) + 1)}
sql_cmd = "SELECT GeneratorExecutable FROM TaskConfiguration WHERE TaskNr == :tasknr"
curc.execute(sql_cmd, data)
res = curc.fetchone()
except:
logmsg = "Failed to fetch Generator Script for Tasknr: {0}".format(tasknr)
logmsg = logmsg + "from the Database! Table TaskConfiguration corrupted?"
c.log_a_msg(self.logger_queue, self.name, logmsg, "ERROR")
finally:
conc.close()
task_start = c.get_task_starttime(self.coursedb, int(tasknr) + 1, self.logger_queue, self.name)
if task_start < datetime.datetime.now():
if res != None: # generator script for this task configured?
logmsg = "Calling Generator Script: " + str(res[0])
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
logmsg = "UserID {0}, UserEmail {1}".format(user_id, user_email)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
self.gen_queue.put(
dict(
{
"user_id": str(user_id),
"user_email": str(user_email),
"task_nr": str(int(tasknr) + 1),
"messageid": "",
}
)
)
else:
c.send_email(
self.sender_queue,
str(user_email),
str(user_id),
"Task",
str(int(tasknr) + 1),
"",
str(message_id),
)
else:
c.send_email(
self.sender_queue,
str(user_email),
str(user_id),
"CurLast",
str(int(tasknr) + 1),
"",
str(message_id),
)
def activator_loop(self):
curc, conc = c.connect_to_db(self.coursedb, self.logger_queue, \
self.name)
# first we need to know, for which tasks, the message has already
# been sent out
sql_cmd = "SELECT * FROM TaskConfiguration WHERE TaskActive==0;"
curc.execute(sql_cmd)
res = curc.fetchone()
while res != None:
tasknr = res[0]
logmsg = "Task {0} is still inactive".format(str(tasknr))
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
# check if a tasks start time has come
task_starttime = datetime.datetime.strptime(res[1], c.format_string)
if task_starttime < datetime.datetime.now():
# first, let's set the task active!
data = {'tasknr': tasknr}
sql_cmd = "UPDATE TaskConfiguration SET TaskActive = 1 WHERE TaskNr == :tasknr;"
curc.execute(sql_cmd, data)
conc.commit()
# next, check if any users are waiting for that task
curs, cons = c.connect_to_db(self.semesterdb, \
self.logger_queue, self.name)
data = {'tasknr': tasknr}
sqlcmd = "SELECT * FROM Users WHERE CurrentTask == :tasknr;"
curs.execute(sqlcmd, data)
nextuser = curs.fetchone()
while nextuser != None:
logmsg = "The next example is sent to User {0} now.".format(tasknr)
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
uid = nextuser[0]
user_email = nextuser[2]
try:
data = {'tasknr': tasknr}
sql_cmd = "SELECT GeneratorExecutable FROM TaskConfiguration WHERE TaskNr == :tasknr;"
curc.execute(sql_cmd, data)
res = curc.fetchone()
except:
logmsg = "Failed to fetch Generator Script for Tasknr: "+ str(tasknr)
logmsg = logmsg + "from the Database! Table TaskConfiguration corrupted?"
c.log_a_msg(self.logger_queue, self.name, \
logmsg, "ERROR")
if res != None:
logmsg = "Calling Generator Script: {0}".format(res[0])
c.log_a_msg(self.logger_queue, self.name, \
logmsg, "DEBUG")
logmsg = "UserID {0}, UserEmail{1}".format(uid, \
user_email)
c.log_a_msg(self.logger_queue, self.name, \
logmsg, "DEBUG")
self.gen_queue.put(dict({"UserId": str(uid), \
"UserEmail": str(user_email), \
"TaskNr": str(tasknr), \
"MessageId": ""}))
else:
c.send_email(self.sender_queue, str(user_email), \
str(uid), "Task", str(tasknr), "", "")
nextuser = curs.fetchone()
cons.close()
res = curc.fetchone()
conc.close()
def generator_loop(self):
"""
Loop code for the generator thread
"""
#blocking wait on gen_queue
next_gen_msg = self.gen_queue.get(True)
logmsg = "gen_queue content:" + str(next_gen_msg)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
task_nr = next_gen_msg.get('task_nr')
user_id = next_gen_msg.get('user_id')
user_email = next_gen_msg.get('user_email')
messageid = next_gen_msg.get('messageid')
#generate the directory for the task
task_dir = 'users/' + str(user_id) + "/Task"+str(task_nr)
c.check_dir_mkdir(task_dir, self.logger_queue, self.name)
#generate the task description
desc_dir = task_dir + "/desc"
c.check_dir_mkdir(desc_dir, self.logger_queue, self.name)
# check if there is a generator executable configured in the database
# if not fall back on static generator script.
curc, conc = c.connect_to_db(self.coursedb, self.logger_queue, self.name)
data = {'TaskNr': task_nr}
sql_cmd = ("SELECT GeneratorExecutable FROM TaskConfiguration "
"WHERE TaskNr== :TaskNr")
curc.execute(sql_cmd, data)
generatorname = curc.fetchone()
if generatorname != None:
data = {'TaskNr': task_nr}
sql_cmd = "SELECT PathToTask FROM TaskConfiguration WHERE TaskNr == :TaskNr"
curc.execute(sql_cmd, data)
path = curc.fetchone()
scriptpath = str(path[0]) + "/" + str(generatorname[0])
else:
scriptpath = "tasks/task" + str(task_nr) + "/./generator.sh"
challenge_mode = self.get_challenge_mode()
command = scriptpath + " " + str(user_id) + " " + str(task_nr) + " " + \
self.submission_email + " " + str(challenge_mode) + " " + \
self.semesterdb + " >> autosub.stdout 2>>autosub.stderr"
logmsg = "generator command: {0}".format(command)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
generator_res = os.system(command)
if generator_res:
logmsg = "Failed to call generator script, return value: " + \
str(generator_res)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
logmsg = "Generated individual task for user/tasknr:" + str(user_id) + "/" + \
str(task_nr)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
c.send_email(self.sender_queue, str(user_email), str(user_id), \
"Task", str(task_nr), "Your personal example", str(messageid))
conc.close()
def test_loop_code(self):
sender_queue = queue.Queue(10)
gen_queue = queue.Queue(10)
ta = TaskActivator(
"testactivator", gen_queue, sender_queue, self.logger_queue, "testcourse.db", "testsemester.db"
)
curc, conc = c.connect_to_db("testcourse.db", self.logger_queue, "testcode")
this_time_yesterday = str(datetime.datetime.now() - datetime.timedelta(1)).split(".")[0]
this_time_tomorrow = str(datetime.datetime.now() + datetime.timedelta(1)).split(".")[0]
this_time_nextmonth = str(datetime.datetime.now() + datetime.timedelta(30)).split(".")[0]
# Drop the task configuration table (start clean)
sqlcmd = "DROP table TaskConfiguration"
try:
curc.execute(sqlcmd)
conc.commit()
except:
logmsg = "No Table TaskConfiguration?"
c.log_a_msg(self.logger_queue, "Debugger", logmsg, "DEBUG")
sqlcmd = (
"CREATE TABLE TaskConfiguration (TaskNr INT PRIMARY KEY, "
"TaskStart DATETIME, TaskDeadline DATETIME, "
"PathToTask TEXT, GeneratorExecutable TEXT, "
"TestExecutable TEXT, Score INT, TaskOperator TEXT, "
"TaskActive BOOLEAN);"
)
curc.execute(sqlcmd)
conc.commit()
# test case 1: no effect -- nothing to do here for the activator thread
self.add_task(
curc,
conc,
1,
this_time_yesterday,
this_time_tomorrow,
"/home/testuser/path/to/task/implementation",
"generator.sh",
"tester.sh",
"5",
"[email protected]",
"1",
)
# test case 2: only a logmsg, there is an inactive task, however its starttime has not
# come (yet)
self.add_task(
curc,
conc,
2,
this_time_tomorrow,
this_time_nextmonth,
"/home/testuser/path/to/task/implementation",
"generator.sh",
"tester.sh",
"5",
"[email protected]",
"0",
)
ta.activator_loop()
# nothing to test for case 1
# check expected log message for test case 2
logmsg = self.logger_queue.get(True)
self.assertEqual(logmsg.get("loggername"), "testactivator")
self.assertEqual(logmsg.get("msg"), "Task 2 is still inactive")
self.assertEqual(logmsg.get("type"), "INFO")
# test case 3: only a logmsg, there is an inactive task, however its starttime has not
# come (yet)
this_time_plus3s = str(datetime.datetime.now() + datetime.timedelta(0, 3)).split(".")[0]
self.add_task(
curc,
conc,
3,
this_time_tomorrow,
this_time_plus3s,
"/home/testuser/path/to/task/implementation",
"generator.sh",
"tester.sh",
"5",
"[email protected]",
"0",
)
ta.activator_loop()
# check expected log message for test case 2
logmsg = self.logger_queue.get(True)
self.assertEqual(logmsg.get("loggername"), "testactivator")
self.assertEqual(logmsg.get("msg"), "Task 2 is still inactive")
self.assertEqual(logmsg.get("type"), "INFO")
# check expected log message for test case 3
logmsg = self.logger_queue.get(True)
self.assertEqual(logmsg.get("loggername"), "testactivator")
self.assertEqual(logmsg.get("msg"), "Task 3 is still inactive")
self.assertEqual(logmsg.get("type"), "INFO")
time.sleep(10)
ta.activator_loop()
# check expected log message for test case 2
logmsg = self.logger_queue.get(True)
self.assertEqual(logmsg.get("loggername"), "testactivator")
self.assertEqual(logmsg.get("msg"), "Task 2 is still inactive")
self.assertEqual(logmsg.get("type"), "INFO")
# check expected log message for test case 3
logmsg = self.logger_queue.get(True)
self.assertEqual(logmsg.get("loggername"), "testactivator")
self.assertEqual(logmsg.get("msg"), "Task 3 is still inactive")
self.assertEqual(logmsg.get("type"), "INFO")
def loop_code(self):
"""
The code run in the while True loop of the mail fetcher thread.
"""
m = self.connect_to_imapserver()
if m != 0:
curs, cons = c.connect_to_db(self.semesterdb, self.logger_queue, self.name)
items = self.fetch_new_emails(m)
# iterate over all new e-mails and take action according to the structure
# of the subject line
for emailid in items:
c.increment_db_statcounter(self.semesterdb, 'nr_mails_fetched', \
self.logger_queue, self.name)
# fetching the mail, "`(RFC822)`" means "get the whole stuff", but you
# can ask for headers only, etc
resp, data = m.fetch(emailid, "(RFC822)")
# parsing the mail content to get a mail object
mail = email.message_from_bytes(data[0][1])
mail_subject = str(mail['subject'])
from_header = str(mail['From'])
split_header = str(from_header).split("<")
user_name = split_header[0]
try:
user_email = str(split_header[1].split(">")[0])
except:
user_email = str(mail['From'])
messageid = mail.get('Message-ID')
whitelisted = self.check_if_whitelisted(user_email)
if whitelisted:
data = {'Email': user_email}
sql_cmd = "SELECT UserId FROM Users WHERE Email = :Email"
curs.execute(sql_cmd, data)
res = curs.fetchall()
if res:
logmsg = "Got mail from an already known user!"
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
#TODO: Does sending a mail "Result bla" without number crash this?
if re.search('[Rr][Ee][Ss][Uu][Ll][Tt]', mail_subject):
searchObj = re.search('[0-9]+', mail_subject, )
if int(searchObj.group()) <= c.get_num_tasks(self.coursedb, \
self.logger_queue, self.name):
logmsg = "Processing a Result, UserId:{0} TaskNr:{1}"\
.format(user_email, searchObj.group())
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
self.take_new_results(user_email, searchObj.group(), \
mail, messageid)
else:
logmsg = ("Given Task number is higher than actual Number"
"of Tasks!")
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
c.send_email(self.sender_queue, user_email, "", \
"InvalidTask", "", "", messageid)
elif re.search('[Qq][Uu][Ee][Ss][Tt][Ii][Oo][Nn]', mail_subject):
self.a_question_was_asked(user_email, mail, messageid)
elif re.search('[Ss][Tt][Aa][Tt][Uu][Ss]', mail_subject):
self.a_status_is_requested(user_email, messageid)
else:
logmsg = ("Got a kind of message I do not understand. "
"Sending a usage mail...")
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
c.send_email(self.sender_queue, user_email, "", "Usage", "", \
"", messageid)
else:
reg_deadline = self.get_registration_deadline()
if reg_deadline > datetime.datetime.now():
self.add_new_user(user_name, user_email)
c.send_email(self.sender_queue, user_email, "", "Welcome", \
"", "", messageid)
else:
c.send_email(self.sender_queue, user_email, "", "RegOver", \
"", "", messageid)
else:
c.send_email(self.sender_queue, user_email, "", "NotAllowed", \
"", "", messageid)
try:
m.close()
except imaplib.IMAP4.abort:
logmsg = ("Closing connection to server was aborted "
"(probably a server-side problem). Trying to connect again ...")
c.log_a_msg(self.logger_queue, self.name, logmsg, "ERROR")
#m.close()
except imaplib.IMAP4.error:
logmsg = ("Got an error when trying to connect to the imap server."
"Trying to connect again ...")
c.log_a_msg(self.logger_queue, self.name, logmsg, "ERROR")
except:
logmsg = ("Got an unknown exception when trying to connect to the "
"imap server. Trying to connect again ...")
c.log_a_msg(self.logger_queue, self.name, logmsg, "ERROR")
finally:
logmsg = "closed connection to imapserver"
c.log_a_msg(self.logger_queue, self.name, logmsg, "INFO")
# check if messages have been handled and need to be archived now
try:
next_send_msg = self.arch_queue.get(False)
except:
next_send_msg = 'NONE'
if next_send_msg != 'NONE':
c.log_a_msg(self.logger_queue, self.name, "moving a message!!!!!!!", \
"INFO")
m = self.connect_to_imapserver()
for next_msg in next_send_msg:
email_ids = self.fetch_all_emails(m)
for emailid in email_ids:
typ, msg_data = m.fetch(str(int(emailid)), "(BODY[HEADER])")
mail = email.message_from_bytes(msg_data[0][1])
if mail['Message-ID'] == next_send_msg.get('mid'):
logmsg = "Moving Message with ID: {0}"\
.format(mail['Message-ID'])
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
resp, data = m.fetch(emailid, "(UID)")
pattern_uid = re.compile('\d+ \(UID (?P<uid>\d+)\)')
match = pattern_uid.match(str(data[0]).split("'")[1])
msg_uid = match.group('uid')
result = m.uid('COPY', msg_uid, 'archive_vels')
if result[0] == 'OK':
mov, data = m.uid('STORE', msg_uid, '+FLAGS', \
'(\Deleted)')
m.expunge()
break
# m==0 is only possible in test-code (e.g. load_test.py)
if m != 0:
m.logout()
cons.close()
time.sleep(self.poll_period)
def handle_next_mail(self):
"""
parse the subject/content of a mail and take appropriate action.
"""
#blocking wait on sender_queue
next_send_msg = self.sender_queue.get(True)
tasknr = str(next_send_msg.get('Task'))
messageid = str(next_send_msg.get('MessageId'))
uid = str(next_send_msg.get('UserId'))
recipient = str(next_send_msg.get('recipient'))
message_type = str(next_send_msg.get('message_type'))
curs, cons = c.connect_to_db(self.semesterdb, self.logger_queue, \
self.name)
curc, conc = c.connect_to_db(self.coursedb, self.logger_queue, \
self.name)
attachments = []
# prepare fields for the e-mail
msg = MIMEMultipart()
msg['From'] = self.mail_user
msg['To'] = recipient
logmsg = "RECIPIENT: " + recipient
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
msg['Date'] = formatdate(localtime=True)
if message_type == "Task":
logmsg = "Task in send_queue: " + str(next_send_msg)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
numtasks = c.get_num_tasks(self.coursedb, \
self.logger_queue, self.name)
ctasknr = c.user_get_current_task(self.semesterdb, uid, \
self.logger_queue, self.name)
if numtasks+1 == int(tasknr): # last task solved!
msg['Subject'] = "Congratulations!"
message_text = self.read_specialmessage('CONGRATS')
if int(tasknr)-1 == int(ctasknr):
# statistics shall only be udated on the first
# successful submission
c.user_set_current_task(self.semesterdb, tasknr, uid, \
self.logger_queue, self.name)
self.increment_db_taskcounter(curs, cons, 'NrSuccessful', \
str(int(tasknr)-1))
self.increment_db_taskcounter(curs, cons, 'NrSubmissions', \
str(int(tasknr)-1))
self.check_and_set_last_done(curs, cons, uid)
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
else: # at least one more task to do: send out the description
# only send the task description, after the first
# successful submission
if int(tasknr)-1 <= int(ctasknr) or int(ctasknr) == 1:
msg['Subject'] = "Description Task" + str(tasknr)
dl_text = "\nDeadline for this Task: {0}\n".format(c.get_task_deadline(self.coursedb, tasknr, self.logger_queue, self.name))
data = {'tasknr': str(tasknr)}
sql_cmd = "SELECT PathToTask FROM TaskConfiguration WHERE TaskNr == :tasknr;"
curc.execute(sql_cmd, data)
paths = curc.fetchone()
if not paths:
logmsg = "It seems, the Path to Task {0} is not configured.".format(tasknr)
c.log_a_msg(self.logger_queue, self.name, \
logmsg, "WARNING")
message_text = "Sorry, but something went wrong... probably misconfiguration or missing configuration of Task {0}".format(tasknr)
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), \
message_type)
else:
path_to_task = str(paths[0])
path_to_msg = path_to_task + "/description.txt"
message_text = self.read_text_file(path_to_msg) \
+ dl_text
data = {'tasknr': str(tasknr), 'uid': uid}
sql_cmd = "SELECT TaskAttachments FROM UserTasks WHERE TaskNr == :tasknr AND UserId == :uid;"
curs.execute(sql_cmd, data)
res = curs.fetchone()
logmsg = "got the following attachments: " + str(res)
c.log_a_msg(self.logger_queue, self.name, logmsg, \
"DEBUG")
if res:
attachments = str(res[0]).split()
# statistics shall only be udated on the first
# succesful submission
c.user_set_current_task(self.semesterdb, tasknr, uid, \
self.logger_queue, self.name)
self.increment_db_taskcounter(curs, cons, \
'NrSuccessful', \
str(int(tasknr)-1))
self.increment_db_taskcounter(curs, cons, \
'NrSubmissions', \
str(int(tasknr)-1))
msg = self.assemble_email(msg, message_text, \
attachments)
self.send_out_email(recipient, msg.as_string(), \
message_type)
self.backup_message(messageid)
elif message_type == "Failed":
self.increment_db_taskcounter(curs, cons, 'NrSubmissions', tasknr)
path_to_msg = "users/{0}/Task{1}".format(uid, tasknr)
error_msg = self.read_text_file("{0}/error_msg".format(path_to_msg))
msg['Subject'] = "Task" + tasknr + ": submission rejected"
message_text = "Error report:\n\n""" + error_msg
reply_attachments = []
try:
logmsg = "searching attachments in: {0}/error_attachments".format(path_to_msg)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
ats = os.listdir("{0}/error_attachments".format(path_to_msg))
logmsg = "got the following attachments: {0}".format(ats)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
for next_attachment in ats:
reply_attachments.append("{0}/error_attachments/{1}".format(path_to_msg, next_attachment))
except:
logmsg = "no attachments for failed task."
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
msg = self.assemble_email(msg, message_text, reply_attachments)
self.send_out_email(recipient, msg.as_string(), message_type)
self.backup_message(messageid)
elif message_type == "SecAlert":
admin_mails = self.get_admin_emails()
for admin_mail in admin_mails:
msg['To'] = admin_mail
path_to_msg = "users/"+ uid + "/Task" + tasknr + "/error_msg"
error_msg = self.read_text_file(path_to_msg)
msg['Subject'] = "Autosub Security Alert User:"******"Error report:\n\n""" + error_msg
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(admin_mail, msg.as_string(), message_type)
self.backup_message(messageid)
elif message_type == "TaskAlert":
admin_mails = self.get_admin_emails()
for admin_mail in admin_mails:
msg['To'] = admin_mail
msg['Subject'] = "Autosub Task Error Alert Task " \
+ tasknr + " User " + uid
message_text = "Something went wrong with task/testbench analyzation for Task " + tasknr +" and User " + uid + " . Either the entity or testbench analyzation threw an error."
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(admin_mail, msg.as_string(), message_type)
self.backup_message(messageid)
elif message_type == "Success":
msg['Subject'] = "Task " + tasknr + " submitted successfully"
message_text = "Congratulations!"
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
#set first done if not set yet
self.check_and_set_first_successful(curs, cons, uid, tasknr)
# no backup of message -- this is done after the new task
# description was sent to the user!
elif message_type == "Status":
msg['Subject'] = "Your Current Status"
message_text = self.generate_status_update(curs, recipient)
numtasks = c.get_num_tasks(self.coursedb, self.logger_queue, \
self.name)
if int(numtasks) >= int(tasknr):
#also attach current task
data = {'tasknr': str(tasknr), 'uid': uid}
sql_cmd = "SELECT TaskAttachments FROM UserTasks WHERE TaskNr == :tasknr AND UserId == :uid;"
curs.execute(sql_cmd, data)
res = curs.fetchone()
logmsg = "got the following attachments: " + str(res)
c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG")
if res:
attachments = str(res[0]).split()
msg = self.assemble_email(msg, message_text, attachments)
self.send_out_email(recipient, msg.as_string(), message_type)
else:
msg = self.assemble_email(msg, message_text, attachments)
self.send_out_email(recipient, msg.as_string(), message_type)
self.backup_message(messageid)
elif message_type == "InvalidTask":
msg['Subject'] = "Invalid Task Number"
message_text = self.read_specialmessage('INVALID')
self.backup_message(messageid)
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
elif message_type == "CurLast":
# we still need to increment the users task counter!
c.user_set_current_task(self.semesterdb, tasknr, uid, \
self.logger_queue, self.name)
msg['Subject'] = "Task{0} is not available yet".format(str(tasknr))
message_text = self.read_specialmessage('CURLAST')
message_text = "{0}\n\nThe Task is currently scheduled for: {1}".format(message_text, \
c.get_task_starttime(self.coursedb, str(tasknr), \
self.logger_queue, self.name))
self.backup_message(messageid)
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
elif message_type == "DeadTask":
msg['Subject'] = "Deadline for Task{0} has passed.".format(str(tasknr))
message_text = self.read_specialmessage('DEADTASK')
self.backup_message(messageid)
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
elif message_type == "Usage":
msg['Subject'] = "Autosub Usage"
message_text = self.read_specialmessage('USAGE')
self.backup_message(messageid)
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
elif message_type == "Question":
msg['Subject'] = "Question received"
message_text = self.read_specialmessage('QUESTION')
self.backup_message(messageid)
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
elif message_type == "QFwd":
orig_mail = next_send_msg.get('Body')
msg['Subject'] = "Question from " + orig_mail['from']
if orig_mail.get_content_maintype() == 'multipart':
part = orig_mail.get_payload(0)
mbody = part.get_payload()
message_text = "Original subject: " + orig_mail['subject'] + "\n\nNote: This e-mail contained attachments which have been removed!\n"
message_text = "{0}\n\nOriginal body:\n{1}".format(message_text, mbody)
else:
mbody = orig_mail.get_payload()
message_text = "Original subject: " + orig_mail['subject'] + \
"\n\nOriginal body:\n" + str(mbody)
self.backup_message(messageid)
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
elif message_type == "Welcome":
msg['Subject'] = "Welcome!"
message_text = self.read_specialmessage('WELCOME')
self.backup_message(messageid)
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
elif message_type == "RegOver":
msg['Subject'] = "Registration Deadline has passed"
message_text = self.read_specialmessage('REGOVER')
self.backup_message(messageid)
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
elif message_type == "NotAllowed":
msg['Subject'] = "Registration Not Successful."
message_text = self.read_specialmessage('NOTALLOWED')
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
self.backup_message(messageid)
else:
c.log_a_msg(self.logger_queue, self.name, \
"Unkown Message Type in the sender_queue!", "ERROR")
msg = self.assemble_email(msg, message_text, '')
self.send_out_email(recipient, msg.as_string(), message_type)
self.backup_message(messageid)
cons.close()
conc.close()
