def test_check_dir_mkdir(self): print("Testing Function: check_dir_mkdir()") self.assertEqual(common.check_dir_mkdir('testdir', self.logger_queue, "testlogger"), 1) self.assertEqual(common.check_dir_mkdir('testdir', self.logger_queue, "testlogger"), 0) os.rmdir('testdir')
def save_submission_user_dir(self, user_id, task_nr, mail): """ Store a new submisson in the user's directory structure. """ # increment the submission_nr for the user submission_nr = self.increment_submission_nr(int(user_id), int(task_nr)) #create a directory for putting his submission in: detach_dir = 'users/{0}/Task{1}'.format(user_id, task_nr) ts = datetime.datetime.now() format_string = '%Y-%m-%d_%H:%M:%S' # e.g 2018-05-01_23:55:21 submission_dir = "/Submission{0}_{1}".format(\ submission_nr, ts.strftime(format_string)) current_dir = detach_dir + submission_dir c.check_dir_mkdir(current_dir, self.queues["logger"], self.name) # use walk to create a generator, iterate on the parts and forget # about the recursive headache for part in mail.walk(): # multipart are just containers, so skip them if part.get_content_maintype() == 'multipart': continue # is this part an attachment ? if part.get('Content-Disposition') is None: continue filename = part.get_filename() counter = 1 # no filename? -> create one with a counter to avoid duplicates if not filename: filename = 'part-%03d%s' % (counter, 'bin') counter += 1 att_path = os.path.join(current_dir, filename) #Check if its already there if not os.path.isfile(att_path): # finally write the stuff fp = open(att_path, 'wb') fp.write(part.get_payload(decode=True)) fp.close() cmd = "rm " + detach_dir + "/*" + " 2> /dev/null" os.system(cmd) cmd = "cp -R " + current_dir + "/* " + detach_dir + " 2> /dev/null" os.system(cmd)
def add_new_user(self, user_name, user_email): """ Add the necessary entries to database for a newly registered user. """ curs, cons = c.connect_to_db(self.semesterdb, self.logger_queue, self.name) logmsg = 'New Account: User: %s' % user_name c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") data = {'Name': user_name, 'Email': user_email, 'TimeNow': str(int(time.time()))} sql_cmd = ("INSERT INTO Users " "(UserId, Name, Email, FirstMail, LastDone, CurrentTask) " "VALUES(NULL, :Name, :Email, datetime(:TimeNow, 'unixepoch', 'localtime')" ", NULL, 1)") curs.execute(sql_cmd, data) cons.commit() # the new user has now been added to the database. Next we need # to send him an email with the first task. # read back the new users UserId and create a directory for putting his # submissions in. data = {'Email': user_email} sql_cmd = "SELECT UserId FROM Users WHERE Email = :Email" curs.execute(sql_cmd, data) res = curs.fetchone() user_id = str(res[0]) dir_name = 'users/'+ user_id c.check_dir_mkdir(dir_name, self.logger_queue, self.name) cons.close() # NOTE: messageid is empty, cause this will be sent out by the welcome message! curc, conc = c.connect_to_db(self.coursedb, self.logger_queue, self.name) sql_cmd = "SELECT GeneratorExecutable FROM TaskConfiguration WHERE TaskNr == 1" curc.execute(sql_cmd) res = curc.fetchone() conc.close() if res != None: logmsg = "Calling Generator Script: " + str(res[0]) c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") logmsg = "UserID " + user_id + ",UserEmail " + user_email c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") self.gen_queue.put(dict({"user_id": user_id, "user_email": user_email, \ "task_nr": "1", "message_id": ""})) else: # If there is no generator script, we assume, that there is a static # description.txt which shall be used. c.send_email(self.sender_queue, user_email, user_id, "Task", "1", "", "")
def check_init_ressources(): """ Check the ressources of autosub if they need to be created and initialized: databases, directories. """ ##################### # SEMESTERDB # ##################### ####### Users ###### fields = ( "UserId INTEGER PRIMARY KEY AUTOINCREMENT, Name TEXT, Email TEXT, " "RegisteredAt DATETIME, LastDone DATETIME, CurrentTask INT") check_and_init_db_table(semesterdb, "Users", fields) ##### TaskStats #### fields = "TaskId INTEGER PRIMARY KEY, NrSubmissions INT, NrSuccessful INT" check_and_init_db_table(semesterdb, "TaskStats", fields) ### StatCounters ### fields = "CounterId INTEGER PRIMARY KEY AUTOINCREMENT, Name TEXT, Value INT" ret = check_and_init_db_table(semesterdb, "StatCounters", fields) if ret: # add the stat counter entries and initialize them to 0: init_db_statvalue('nr_mails_fetched', 0) init_db_statvalue('nr_mails_sent', 0) init_db_statvalue('nr_questions_received', 0) init_db_statvalue('nr_non_registered', 0) init_db_statvalue('nr_status_requests', 0) ##### UserTasks #### fields = ("UniqueId INTEGER PRIMARY KEY AUTOINCREMENT, TaskNr INT, " "UserId INT, TaskParameters TEXT, TaskDescription TEXT, " "TaskAttachments TEXT, NrSubmissions INTEGER, " "FirstSuccessful INTEGER") ret = check_and_init_db_table(semesterdb, "UserTasks", fields) ##### SuccessfulTasks #### fields = "UserId INTEGER, TaskNr INTEGER, PRIMARY KEY (UserId, TaskNr)" ret = check_and_init_db_table(semesterdb, "SuccessfulTasks", fields) #### Whitelist ##### fields = "UniqueId INTEGER PRIMARY KEY AUTOINCREMENT, Email TEXT UNIQUE, Name TEXT" ret = check_and_init_db_table(semesterdb, "Whitelist", fields) #################### # DIRECTORY USERS # #################### c.check_dir_mkdir("users", logger_queue, "autosub.py") ##################### # COURSEDB # ##################### # TaskConfiguration # fields = ( "TaskNr INT PRIMARY KEY, TaskStart DATETIME, " "TaskDeadline DATETIME, TaskName TEXT, GeneratorExecutable TEXT, " "Language TEXT, " "TestExecutable TEXT, BackendInterfaceFile TEXT, Score INT, " "TaskOperator TEXT, TaskActive BOOLEAN") ret = check_and_init_db_table(coursedb, "TaskConfiguration", fields) ## SpecialMessages # fields = "EventName TEXT PRIMARY KEY, EventText TEXT" check_and_init_db_table(coursedb, "SpecialMessages", fields) # we use the .txt files to set the SpecialMessages in the database with jinja2 env = Environment() env.loader = FileSystemLoader(specialmsgs_dir) filename = 'welcome.txt' load_specialmessage_to_db(env, 'WELCOME', filename) filename = 'usage.txt' load_specialmessage_to_db(env, 'USAGE', filename) filename = 'question.txt' load_specialmessage_to_db(env, 'QUESTION', filename) filename = 'invalidtask.txt' load_specialmessage_to_db(env, 'INVALID', filename) filename = 'congratulations.txt' load_specialmessage_to_db(env, 'CONGRATS', filename) filename = 'registrationover.txt' load_specialmessage_to_db(env, 'REGOVER', filename) filename = 'notallowed.txt' load_specialmessage_to_db(env, 'NOTALLOWED', filename) filename = 'deletedfromwhitelist.txt' load_specialmessage_to_db(env, 'DeletedFromWhitelist', filename) filename = 'curlast.txt' load_specialmessage_to_db(env, 'CURLAST', filename) filename = 'deadtask.txt' load_specialmessage_to_db(env, 'DEADTASK', filename) filename = 'tasknotsubmittable.txt' load_specialmessage_to_db(env, 'TASKNOTSUBMITTABLE', filename) filename = 'tasknotactive.txt' load_specialmessage_to_db(env, 'TASKNOTACTIVE', filename) filename = 'nomultiplerequest.txt' load_specialmessage_to_db(env, 'NOMULTIPLEREQUEST', filename) ### GeneralConfig ## fields = "ConfigItem Text PRIMARY KEY, Content TEXT" ret = check_and_init_db_table(coursedb, "GeneralConfig", fields) if ret: # default values for first time, these values can be changed during # operation set_general_config_param('registration_deadline', 'NULL') set_general_config_param('archive_dir', 'archive') set_general_config_param('admin_email', '') # values that are read and replaced at every new start of autosub set_general_config_param('course_mode', course_mode) set_general_config_param('course_name', course_name) set_general_config_param('submission_email', imapmail) set_general_config_param('tasks_dir', tasks_dir) set_general_config_param('auto_advance', auto_advance) set_general_config_param('allow_requests', allow_requests)
def add_new_user(self, user_name, user_email): """ Add the necessary entries to database for a newly registered user. Call generator thread to create the first task. """ curs, cons = c.connect_to_db(self.dbs["semester"], self.queues["logger"], self.name) logmsg = 'Creating new Account: User: %s' % user_name c.log_a_msg(self.queues["logger"], self.name, logmsg, "INFO") data = { 'Name': user_name, 'Email': user_email, 'TimeNow': str(int(time.time())) } sql_cmd = ( "INSERT INTO Users " "(UserId, Name, Email, RegisteredAt, LastDone, CurrentTask) " "VALUES(NULL, :Name, :Email, datetime(:TimeNow, 'unixepoch', 'localtime')" ", NULL, 1)") curs.execute(sql_cmd, data) cons.commit() # read back the new users UserId and create a directory for putting his # submissions in. data = {'Email': user_email} sql_cmd = "SELECT UserId FROM Users WHERE Email = :Email" curs.execute(sql_cmd, data) res = curs.fetchone() if res is None: logmsg = ("Created new user with " "name= {0} , email={1} failed").format( user_name, user_email) c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG") return user_id = str(res[0]) dir_name = 'users/' + user_id c.check_dir_mkdir(dir_name, self.queues["logger"], self.name) cons.close() # in allow_requests mode the user gets no first task send if self.allow_requests != "no": return # Give the user the task which is starttime <= now < deadline AND # min(TaskNr) as initial task curc, conc = c.connect_to_db(self.dbs["course"], self.queues["logger"], self.name) data = {'TimeNow': str(int(time.time()))} sql_cmd = ( "SELECT MIN(TaskNr) FROM TaskConfiguration " "WHERE TaskStart <= datetime(:TimeNow, 'unixepoch','localtime') AND " "TaskDeadline > datetime(:TimeNow, 'unixepoch', 'localtime')") curc.execute(sql_cmd, data) res = curc.fetchone() conc.close() if not res or not res[0]: logmsg = ( "Error generating initial Task for UserId = {0}. Could not " "find a initial task for this user").format(user_id) c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR") return task_nr = int(res[0]) #adjust users CurrentTask if he does not get Task with task_nr=1 if task_nr != 1: c.user_set_current_task(self.dbs["semester"], task_nr, user_id, \ self.queues["logger"], self.name) logmsg = ("Calling Generator to create" "TaskNr:{0} for UserId:{1}").format(task_nr, user_id) c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG") c.generate_task(self.queues["generator"], user_id, task_nr, user_email, "")
def generator_loop(self): """ Loop code for the generator thread """ #blocking wait on gen_queue next_gen_msg = self.gen_queue.get(True) logmsg = "gen_queue content:" + str(next_gen_msg) c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") task_nr = next_gen_msg.get('task_nr') user_id = next_gen_msg.get('user_id') user_email = next_gen_msg.get('user_email') messageid = next_gen_msg.get('messageid') #generate the directory for the task task_dir = 'users/' + str(user_id) + "/Task" + str(task_nr) c.check_dir_mkdir(task_dir, self.logger_queue, self.name) #generate the task description desc_dir = task_dir + "/desc" c.check_dir_mkdir(desc_dir, self.logger_queue, self.name) # check if there is a generator executable configured in the database # if not fall back on static generator script. curc, conc = c.connect_to_db(self.coursedb, self.logger_queue, self.name) data = {'TaskNr': task_nr} sql_cmd = ("SELECT GeneratorExecutable FROM TaskConfiguration " "WHERE TaskNr== :TaskNr") curc.execute(sql_cmd, data) generatorname = curc.fetchone() if generatorname != None: data = {'TaskNr': task_nr} sql_cmd = "SELECT PathToTask FROM TaskConfiguration WHERE TaskNr == :TaskNr" curc.execute(sql_cmd, data) path = curc.fetchone() scriptpath = str(path[0]) + "/" + str(generatorname[0]) else: scriptpath = "tasks/task" + str(task_nr) + "/./generator.sh" challenge_mode = self.get_challenge_mode() command = scriptpath + " " + str(user_id) + " " + str(task_nr) + " " + \ self.submission_email + " " + str(challenge_mode) + " " + \ self.semesterdb + " >> autosub.stdout 2>>autosub.stderr" logmsg = "generator command: {0}".format(command) c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") generator_res = os.system(command) if generator_res: logmsg = "Failed to call generator script, return value: " + \ str(generator_res) c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") logmsg = "Generated individual task for user/tasknr:" + str(user_id) + "/" + \ str(task_nr) c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") c.send_email(self.sender_queue, str(user_email), str(user_id), \ "Task", str(task_nr), "Your personal example", str(messageid)) conc.close()
def init_ressources(semesterdb, coursedb, num_tasks, subsmission_email, challenge_mode, \ course_name, special_path, allow_skipping): """ Check if all databases, tables, etc. are available, or if they have to be created. If non-existent --> create them. """ # needed for some actions after table creation curs, cons = c.connect_to_db(semesterdb, logger_queue, "autosub.py") #################### ####### Users ###### #################### fields = ( "UserId INTEGER PRIMARY KEY AUTOINCREMENT, Name TEXT, Email TEXT, " "FirstMail DATETIME, LastDone DATETIME, CurrentTask INT") check_and_init_db_table(semesterdb, "Users", fields) #################### ##### TaskStats #### #################### fields = "TaskId INTEGER PRIMARY KEY, NrSubmissions INT, NrSuccessful INT" ret = check_and_init_db_table(semesterdb, "TaskStats", fields) if ret: for t in range(1, num_tasks + 1): data = {'TaskId': t} sql_cmd = ( "INSERT INTO TaskStats (TaskId, NrSubmissions, NrSuccessful) " "VALUES(:TaskId, 0, 0)") curs.execute(sql_cmd, data) cons.commit() #################### ### StatCounters ### #################### fields = "CounterId INTEGER PRIMARY KEY AUTOINCREMENT, Name TEXT, Value INT" ret = check_and_init_db_table(semesterdb, "StatCounters", fields) if ret: # add the stat counter entries and initialize them to 0: init_db_statvalue(semesterdb, 'nr_mails_fetched', 0) init_db_statvalue(semesterdb, 'nr_mails_sent', 0) init_db_statvalue(semesterdb, 'nr_questions_received', 0) init_db_statvalue(semesterdb, 'nr_non_registered', 0) init_db_statvalue(semesterdb, 'nr_status_requests', 0) #################### ##### UserTasks #### #################### fields = ( "UniqueId INTEGER PRIMARY KEY AUTOINCREMENT, TaskNr INT, UserId INT, " "TaskParameters TEXT, TaskDescription TEXT, TaskAttachments TEXT, " "NrSubmissions INTEGER, FirstSuccessful INTEGER") ret = check_and_init_db_table(semesterdb, "UserTasks", fields) #################### #### Whitelist ##### #################### fields = "UniqueId INTEGER PRIMARY KEY AUTOINCREMENT, Email TEXT" ret = check_and_init_db_table(semesterdb, "Whitelist", fields) #################### # Directory users ## #################### c.check_dir_mkdir("users", logger_queue, "autosub.py") # we don't need semesterdb connection anymore cons.close() #################### ## SpecialMessages # #################### fields = "EventName TEXT PRIMARY KEY, EventText TEXT" ret = check_and_init_db_table(coursedb, "SpecialMessages", fields) # that table did not exists, therefore we use the .txt files to initialize it! if ret: if allow_skipping == True: filename = '{0}SpecialMessages/welcome_withskip.txt'.format( special_path) load_specialmessage_to_db(coursedb, 'WELCOME', filename, subsmission_email, \ course_name) filename = '{0}SpecialMessages/usage_withskip.txt'.format( special_path) load_specialmessage_to_db(coursedb, 'USAGE', filename, subsmission_email, \ course_name) else: filename = '{0}SpecialMessages/welcome.txt'.format(special_path) load_specialmessage_to_db(coursedb, 'WELCOME', filename, subsmission_email, \ course_name) filename = '{0}SpecialMessages/usage.txt'.format(special_path) load_specialmessage_to_db(coursedb, 'USAGE', filename, subsmission_email, \ course_name) filename = '{0}SpecialMessages/question.txt'.format(special_path) load_specialmessage_to_db(coursedb, 'QUESTION', filename, subsmission_email, \ course_name) filename = '{0}SpecialMessages/invalidtask.txt'.format(special_path) load_specialmessage_to_db(coursedb, 'INVALID', filename, subsmission_email, \ course_name) filename = '{0}SpecialMessages/congratulations.txt'.format( special_path) load_specialmessage_to_db(coursedb, 'CONGRATS', filename, subsmission_email, \ course_name) filename = '{0}SpecialMessages/registrationover.txt'.format( special_path) load_specialmessage_to_db(coursedb, 'REGOVER', filename, subsmission_email, \ course_name) filename = '{0}SpecialMessages/notallowed.txt'.format(special_path) load_specialmessage_to_db(coursedb, 'NOTALLOWED', filename, subsmission_email, \ course_name) filename = '{0}SpecialMessages/curlast.txt'.format(special_path) load_specialmessage_to_db(coursedb, 'CURLAST', filename, subsmission_email, \ course_name) filename = '{0}SpecialMessages/deadtask.txt'.format(special_path) load_specialmessage_to_db(coursedb, 'DEADTASK', filename, subsmission_email, \ course_name) filename = '{0}SpecialMessages/skipnotpossible.txt'.format( special_path) load_specialmessage_to_db(coursedb, 'SKIPNOTPOSSIBLE', filename, subsmission_email, \ course_name) filename = '{0}SpecialMessages/tasknotsubmittable.txt'.format( special_path) load_specialmessage_to_db(coursedb, 'TASKNOTSUBMITTABLE', filename, subsmission_email, \ course_name) ##################### # TaskConfiguration # ##################### fields = ( "TaskNr INT PRIMARY KEY, TaskStart DATETIME, TaskDeadline DATETIME, " "PathToTask TEXT, GeneratorExecutable TEXT, TestExecutable TEXT, " "Score INT, TaskOperator TEXT, TaskActive BOOLEAN") ret = check_and_init_db_table(coursedb, "TaskConfiguration", fields) #################### ### GeneralConfig ## #################### fields = "ConfigItem Text PRIMARY KEY, Content TEXT" ret = check_and_init_db_table(coursedb, "GeneralConfig", fields) ##################### # Num workers,tasks # ##################### # if that table did not exist, load the defaults given in the configuration file if ret: set_general_config_param(coursedb, 'num_tasks', str(num_tasks)) set_general_config_param(coursedb, 'registration_deadline', 'NULL') set_general_config_param(coursedb, 'archive_dir', 'archive') set_general_config_param(coursedb, 'admin_email', '') set_general_config_param(coursedb, 'challenge_mode', challenge_mode) set_general_config_param(coursedb, 'course_name', course_name)
def take_new_results(self, user_email, task_nr, mail, messageid): """ Store a new submisson in the user's directory structure. """ curs, cons = c.connect_to_db(self.semesterdb, self.logger_queue, self.name) data = {'Email': user_email} sql_cmd = "SELECT UserId FROM Users WHERE Email = :Email" curs.execute(sql_cmd, data) res = curs.fetchone() user_id = res[0] deadline = c.get_task_deadline(self.coursedb, task_nr, self.logger_queue, \ self.name) curtask = c.user_get_current_task(self.semesterdb, user_id, self.logger_queue, \ self.name) if deadline < datetime.datetime.now(): #deadline has passed! c.send_email(self.sender_queue, user_email, "", "DeadTask", str(task_nr), \ "", messageid) elif curtask < task_nr: #user is trying to submit a solution to a task although an earlier task # was not solved. c.send_email(self.sender_queue, user_email, "", "InvalidTask", str(task_nr), \ "", messageid) else: # get the user's UserId data = {'Email': user_email} sql_cmd = "SELECT UserId FROM Users WHERE Email = :Email" curs.execute(sql_cmd, data) res = curs.fetchone() user_id = res[0] # increment the submission_nr for the user submission_nr = self.increment_submission_nr(int(user_id), int(task_nr)) #create a directory for putting his submission in: detach_dir = 'users/{0}/Task{1}'.format(user_id, task_nr) ts = datetime.datetime.now() submission_dir = "/Submission{0}_{1}{2}{3}_{4}{5}{6}{7}".format(\ submission_nr, ts.year, ts.month, ts.day, ts.hour, ts.minute, \ ts.second, ts.microsecond) current_dir = detach_dir + submission_dir c.check_dir_mkdir(current_dir, self.logger_queue, self.name) # use walk to create a generator, iterate on the parts and forget # about the recursive headache for part in mail.walk(): # multipart are just containers, so skip them if part.get_content_maintype() == 'multipart': continue # is this part an attachment ? if part.get('Content-Disposition') is None: continue filename = part.get_filename() counter = 1 # if there is no filename, create one with a counter to avoid duplicates if not filename: filename = 'part-%03d%s' % (counter, 'bin') counter += 1 att_path = os.path.join(current_dir, filename) #Check if its already there if not os.path.isfile(att_path): # finally write the stuff fp = open(att_path, 'wb') fp.write(part.get_payload(decode=True)) fp.close() cmd = "rm " + detach_dir + "/*" + " 2> /dev/null" os.system(cmd) cmd = "cp -R " + current_dir + "/* " + detach_dir + " > /dev/null" os.system(cmd) # Next, let's handle the task that shall be checked, and send a job # request to the job_queue. The workers can then get it from there. self.job_queue.put(dict({"UserId": user_id, "UserEmail": user_email, \ "message_type": "Task", "taskNr": task_nr, \ "MessageId": messageid})) cons.close()
def generator_loop(self): """ Loop code for the generator thread """ # blocking wait on gen_queue next_gen_msg = self.queues["generator"].get(True) logmsg = "gen_queue content:" + str(next_gen_msg) c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG") task_nr = next_gen_msg.get('task_nr') user_id = next_gen_msg.get('user_id') user_email = next_gen_msg.get('user_email') message_id = next_gen_msg.get('message_id') # requested task is a valid task? if not c.is_valid_task_nr(self.dbs["course"], task_nr, \ self.queues["logger"], self.name): logmsg = ( "Generator was given the task to create non valid TaskNr {0}. " "This should not happen!").format(task_nr) c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR") return # check if user already got this task already_received = c.user_received_task(self.dbs["semester"], user_id, \ task_nr, self.queues["logger"], \ self.name) if already_received: if self.allow_requests == "multiple": logmsg = ("User with Id {0} TaskNr {1} already got this task, " "deleting it to make place for new").format( user_id, task_nr) c.log_a_msg(self.queues["logger"], self.name, logmsg, "INFO") self.delete_usertask(user_id, task_nr) else: logmsg = ( "User with Id {0} TaskNr {1} already got this task, " "multiple request not allowed for this course").format( user_id, task_nr) c.log_a_msg(self.queues["logger"], self.name, logmsg, "INFO") c.send_email(self.queues["sender"], str(user_email), str(user_id), \ "NoMultipleRequest", str(task_nr), "", str(message_id)) return # generate the directory for the task in the space of the user usertask_dir = 'users/' + str(user_id) + "/Task" + str(task_nr) c.check_dir_mkdir(usertask_dir, self.queues["logger"], self.name) # generate the folder for the task description desc_dir = usertask_dir + "/desc" c.check_dir_mkdir(desc_dir, self.queues["logger"], self.name) # get the path to the generator script scriptpath, language = self.get_scriptinfo(task_nr) # check the path if not scriptpath or not os.path.isfile(scriptpath): logmsg = "Could not find generator script for task{0}".format( task_nr) c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR") return command = [scriptpath, str(user_id), str(task_nr), self.submission_mail,\ str(self.course_mode), self.dbs["semester"], str(language)] logmsg = "generator command with arguments: {0} ".format(command) c.log_a_msg(self.queues["logger"], self.name, logmsg, "DEBUG") process = Popen(command, stdout=PIPE, stderr=PIPE) generator_msg, generator_error = process.communicate() generator_msg = generator_msg.decode('UTF-8') generator_error = generator_error.decode('UTF-8') generator_res = process.returncode log_src = "Generator{0}({1})".format(str(task_nr), str(user_id)) if generator_msg: c.log_task_msg(self.queues["logger"], log_src, generator_msg, "INFO") if generator_error: c.log_task_error(self.queues["logger"], log_src, generator_error, "ERROR") # Error at task generation if generator_res != 0: # generator not 0 returned logmsg = "Failed executing the generator script, return value: " + \ str(generator_res) c.log_a_msg(self.queues["logger"], self.name, logmsg, "ERROR") # alert to admin c.send_email(self.queues["sender"], "", user_id, \ "TaskAlert", task_nr, "", message_id) #notice to user c.send_email(self.queues["sender"], user_email, user_id, \ "TaskErrorNotice", task_nr, "", message_id) return logmsg = "Generated individual task for user/task_nr:" + str(user_id) \ + "/" + str(task_nr) c.log_a_msg(self.queues["logger"], self.name, logmsg, "INFO") c.send_email(self.queues["sender"], str(user_email), str(user_id), \ "Task", str(task_nr), "Your personal example", str(message_id))
def save_submission_user_dir(self, user_email, task_nr, mail, messageid): """ Store a new submisson in the user's directory structure. """ curs, cons = c.connect_to_db(self.semesterdb, self.logger_queue, self.name) data = {'Email': user_email} sql_cmd = "SELECT UserId FROM Users WHERE Email = :Email" curs.execute(sql_cmd, data) res = curs.fetchone() user_id = res[0] # increment the submission_nr for the user submission_nr = self.increment_submission_nr(int(user_id), int(task_nr)) #create a directory for putting his submission in: detach_dir = 'users/{0}/Task{1}'.format(user_id, task_nr) ts = datetime.datetime.now() submission_dir = "/Submission{0}_{1}{2}{3}_{4}{5}{6}{7}".format(\ submission_nr, ts.year, ts.month, ts.day, ts.hour, ts.minute, \ ts.second, ts.microsecond) current_dir = detach_dir + submission_dir c.check_dir_mkdir(current_dir, self.logger_queue, self.name) # use walk to create a generator, iterate on the parts and forget # about the recursive headache for part in mail.walk(): # multipart are just containers, so skip them if part.get_content_maintype() == 'multipart': continue # is this part an attachment ? if part.get('Content-Disposition') is None: continue filename = part.get_filename() counter = 1 # if there is no filename, create one with a counter to avoid duplicates if not filename: filename = 'part-%03d%s' % (counter, 'bin') counter += 1 att_path = os.path.join(current_dir, filename) #Check if its already there if not os.path.isfile(att_path): # finally write the stuff fp = open(att_path, 'wb') fp.write(part.get_payload(decode=True)) fp.close() cmd = "rm " + detach_dir + "/*" + " 2> /dev/null" os.system(cmd) cmd = "cp -R " + current_dir + "/* " + detach_dir + " 2> /dev/null" os.system(cmd) cons.close()
def add_new_user(self, user_name, user_email): """ Add the necessary entries to database for a newly registered user. Call generator thread to create the first task. """ curs, cons = c.connect_to_db(self.semesterdb, self.logger_queue, self.name) logmsg = 'Creating new Account: User: %s' % user_name c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") data = { 'Name': user_name, 'Email': user_email, 'TimeNow': str(int(time.time())) } sql_cmd = ( "INSERT INTO Users " "(UserId, Name, Email, FirstMail, LastDone, CurrentTask) " "VALUES(NULL, :Name, :Email, datetime(:TimeNow, 'unixepoch', 'localtime')" ", NULL, 1)") curs.execute(sql_cmd, data) cons.commit() # the new user has now been added to the database. Next we need # to send him an email with the first task. # read back the new users UserId and create a directory for putting his # submissions in. data = {'Email': user_email} sql_cmd = "SELECT UserId FROM Users WHERE Email = :Email" curs.execute(sql_cmd, data) res = curs.fetchone() if res == None: logmsg = ("Created new user with " "name= {0} , email={1} failed").format( user_name, user_email) c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") user_id = str(res[0]) dir_name = 'users/' + user_id c.check_dir_mkdir(dir_name, self.logger_queue, self.name) cons.close() # Give the user the task which is starttime <= now < deadline AND # min(TaskNr) curc, conc = c.connect_to_db(self.coursedb, self.logger_queue, self.name) data = {'TimeNow': str(int(time.time()))} sql_cmd = ( "SELECT MIN(TaskNr) FROM TaskConfiguration " "WHERE TaskStart <= datetime(:TimeNow, 'unixepoch','localtime') AND " "TaskDeadline > datetime(:TimeNow, 'unixepoch', 'localtime')") curc.execute(sql_cmd, data) res = curc.fetchone() conc.close() if res == None: logmsg = ( "Error generating first Task for UserId = {0}. Could not " "find first task for this user").format(user_id) c.log_a_msg(self.logger_queue, self.name, logmsg, "ERROR") else: task_nr = int(res[0]) #adjust users CurrentTask if he does not get Task with task_nr=1 if task_nr != 1: c.user_set_current_task(self.semesterdb, task_nr, user_id, \ self.logger_queue, self.name) logmsg = ("Calling Generator to create" "TaskNr:{0} for UserId:{1}").format(task_nr, user_id) c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") self.gen_queue.put(dict({"user_id": user_id, "user_email": user_email, \ "task_nr": task_nr, "messageid": ""}))
def generator_loop(self): """ Loop code for the generator thread """ #blocking wait on gen_queue next_gen_msg = self.gen_queue.get(True) logmsg = "gen_queue content:" + str(next_gen_msg) c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") task_nr = next_gen_msg.get('task_nr') user_id = next_gen_msg.get('user_id') user_email = next_gen_msg.get('user_email') messageid = next_gen_msg.get('messageid') #generate the directory for the task task_dir = 'users/' + str(user_id) + "/Task"+str(task_nr) c.check_dir_mkdir(task_dir, self.logger_queue, self.name) #generate the task description desc_dir = task_dir + "/desc" c.check_dir_mkdir(desc_dir, self.logger_queue, self.name) # check if there is a generator executable configured in the database # if not fall back on static generator script. curc, conc = c.connect_to_db(self.coursedb, self.logger_queue, self.name) data = {'TaskNr': task_nr} sql_cmd = ("SELECT GeneratorExecutable FROM TaskConfiguration " "WHERE TaskNr== :TaskNr") curc.execute(sql_cmd, data) generatorname = curc.fetchone() if generatorname != None: data = {'TaskNr': task_nr} sql_cmd = "SELECT PathToTask FROM TaskConfiguration WHERE TaskNr == :TaskNr" curc.execute(sql_cmd, data) path = curc.fetchone() scriptpath = str(path[0]) + "/" + str(generatorname[0]) else: scriptpath = "tasks/task" + str(task_nr) + "/./generator.sh" challenge_mode = self.get_challenge_mode() command = scriptpath + " " + str(user_id) + " " + str(task_nr) + " " + \ self.submission_email + " " + str(challenge_mode) + " " + \ self.semesterdb + " >> autosub.stdout 2>>autosub.stderr" logmsg = "generator command: {0}".format(command) c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") generator_res = os.system(command) if generator_res: logmsg = "Failed to call generator script, return value: " + \ str(generator_res) c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") logmsg = "Generated individual task for user/tasknr:" + str(user_id) + "/" + \ str(task_nr) c.log_a_msg(self.logger_queue, self.name, logmsg, "DEBUG") c.send_email(self.sender_queue, str(user_email), str(user_id), \ "Task", str(task_nr), "Your personal example", str(messageid)) conc.close()