def test_invalid_type_write_submitty_date(self): testcases = ('2020-06-12 03:21:30.123+0000', 10) for testcase in testcases: with self.subTest(testcase): with self.assertRaises(TypeError) as cm: dateutils.write_submitty_date(10) self.assertEqual( "Invalid type. Expected datetime or datetime string," " got <class 'int'>.", str(cm.exception))
def log_stack_trace(job_id="UNKNOWN", is_batch=False, which_untrusted="", jobname="", timelabel="", elapsed_time=-1, trace=""): now = dateutils.get_current_time() datefile = "stack_traces_{0}.txt".format(datetime.strftime(now, "%Y%m%d")) autograding_log_file = os.path.join(AUTOGRADING_STACKTRACE_PATH, datefile) easy_to_read_date = dateutils.write_submitty_date(now, True) batch_string = "BATCH" if is_batch else "" if elapsed_time == "": elapsed_time = -1 elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format( elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" with open(autograding_log_file, 'a') as myfile: try: fcntl.flock(myfile, fcntl.LOCK_EX | fcntl.LOCK_NB) print("%s | %6s | %5s | %11s | %-75s | %-6s %9s %3s |\n%s" % (easy_to_read_date, job_id, batch_string, which_untrusted, jobname, timelabel, elapsed_time_string, time_unit, trace), file=myfile) fcntl.flock(myfile, fcntl.LOCK_UN) except: print("Could not gain a lock on the log file.")
def test_write_submitty_date(self, get_timezone): testcases = ((datetime(2020, 6, 12, 3, 21, 30, tzinfo=pytz_timezone('UTC')), '2020-06-12 03:21:30+0000'), (datetime(2020, 12, 25, 3, 21, 30, tzinfo=pytz_timezone('UTC')), '2020-12-25 03:21:30+0000'), (datetime(2020, 6, 12, 3, 21, 30, 123, tzinfo=pytz_timezone('UTC')), '2020-06-12 03:21:30+0000'), (datetime(2020, 6, 12, 3, 21, 30), '2020-06-12 03:21:30-0400'), (datetime(2020, 12, 12, 3, 21, 30), '2020-12-12 03:21:30-0500')) for testcase in testcases: with self.subTest(i=testcase[0]): self.assertEqual(testcase[1], dateutils.write_submitty_date(testcase[0]))
def log_message(log_path, job_id="UNKNOWN", is_batch=False, which_untrusted="", jobname="", timelabel="", elapsed_time=-1, message=""): """ Given a log directory, create or append a message to a dated log file in that directory. """ now = dateutils.get_current_time() datefile = datetime.strftime(now, "%Y%m%d") + ".txt" autograding_log_file = os.path.join(log_path, datefile) easy_to_read_date = dateutils.write_submitty_date(now, True) batch_string = "BATCH" if is_batch else "" if elapsed_time == "": elapsed_time = -1 elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format( elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" parts = (easy_to_read_date, f"{job_id:>6s}", f"{batch_string:>5s}", f"{which_untrusted:>11s}", f"{jobname:75s}", f"{timelabel:6s} {elapsed_time_string:>9s} {time_unit:>3s}", message) write_to_log(autograding_log_file, parts)
def log_message(log_path, job_id="UNKNOWN", is_batch=False, which_untrusted="", jobname="", timelabel="", elapsed_time=-1, message=""): """ Given a log directory, create or append a message to a dated log file in that directory. """ now = dateutils.get_current_time() datefile = datetime.strftime(now, "%Y%m%d") + ".txt" autograding_log_file = os.path.join(log_path, datefile) easy_to_read_date = dateutils.write_submitty_date(now, True) batch_string = "BATCH" if is_batch else "" if elapsed_time == "": elapsed_time = -1 elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format( elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" with open(autograding_log_file, 'a') as myfile: try: fcntl.flock(myfile, fcntl.LOCK_EX | fcntl.LOCK_NB) print( "%s | %6s | %5s | %11s | %-75s | %-6s %9s %3s | %s" % (easy_to_read_date, job_id, batch_string, which_untrusted, jobname, timelabel, elapsed_time_string, time_unit, message), file=myfile) fcntl.flock(myfile, fcntl.LOCK_UN) except: print("Could not gain a lock on the log file.")
def log_container_meta(log_path, event="", name="", container="", time=0): """ Given a log file, create or append container meta data to a log file. """ now = dateutils.get_current_time() easy_to_read_date = dateutils.write_submitty_date(now, True) time_unit = "sec" parts = (easy_to_read_date, name, container, event, f"{time:.3f}", time_unit) write_to_log(log_path, parts)
def main(): args = parse_args() complete_config_json_path = os.path.join(SUBMITTY_DATA_DIR, 'courses', args.semester, args.course, 'config', 'complete_config', 'complete_config_' + args.assignment + '.json') if os.path.isfile(complete_config_json_path): with open(complete_config_json_path,'r', encoding='utf-8') as infile: config_file=json.load(infile) else: sys.exit(1) required_capabilities = config_file.get('required_capabilities','default') testcases = config_file.get('testcases',[]) graded_file = { "semester": args.semester, "course": args.course, "gradeable": args.assignment, "required_capabilities": required_capabilities, "queue_time": dateutils.write_submitty_date(microseconds=True), "generate_output": True, "max_possible_grading_time" : -1, "who" : "build", "regrade" : False, } should_generated_output = False for testcase in testcases: input_generation_commands = testcase.get('input_generation_commands',[]) solution_containers = testcase.get('solution_containers',[]) should_generate_solution = False for solution_container in solution_containers: if len(solution_container["commands"]) != 0 : should_generate_solution = True break if should_generate_solution and not input_generation_commands: path_grading_file = os.path.join(SUBMITTY_DATA_DIR, "to_be_graded_queue", "__".join([args.semester, args.course, args.assignment])) if os.path.isfile(path_grading_file): os.remove(path_grading_file) with open(path_grading_file, 'w') as grading_file: json.dump(graded_file, grading_file,sort_keys=True,indent=4) print("Starting to build generated output") break
def worker_process(which_machine,address,which_untrusted,my_server): # verify the DAEMON_USER is running this script if not int(os.getuid()) == int(DAEMON_UID): grade_items_logging.log_message(JOB_ID, message="ERROR: must be run by DAEMON_USER") raise SystemExit("ERROR: the grade_item.py script must be run by the DAEMON_USER") # ignore keyboard interrupts in the worker processes signal.signal(signal.SIGINT, signal.SIG_IGN) counter = 0 servername_workername = "{0}_{1}".format(my_server, address) autograding_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_autograding.zip") submission_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_submission.zip") todo_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_queue.json") while True: if os.path.exists(todo_queue_file): try: results_zip_tmp = grade_item.grade_from_zip(autograding_zip,submission_zip,which_untrusted) results_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_results.zip") done_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_queue.json") #move doesn't inherit the permissions of the destination directory. Copyfile does. shutil.copyfile(results_zip_tmp, results_zip) os.remove(results_zip_tmp) with open(todo_queue_file, 'r') as infile: queue_obj = json.load(infile) queue_obj["done_time"]=dateutils.write_submitty_date(microseconds=True) with open(done_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) except Exception as e: grade_items_logging.log_message(JOB_ID, message="ERROR attempting to unzip graded item: " + which_machine + " " + which_untrusted + " exception " + repr(e)) with contextlib.suppress(FileNotFoundError): os.remove(autograding_zip) with contextlib.suppress(FileNotFoundError): os.remove(submission_zip) with contextlib.suppress(FileNotFoundError): os.remove(todo_queue_file) counter = 0 else: if counter >= 10: print (which_machine,which_untrusted,"wait") counter = 0 counter += 1 time.sleep(1)
def log_message(is_batch, which_untrusted, jobname, timelabel, elapsed_time, message): now = dateutils.get_current_time() datefile = datetime.strftime(now, "%Y%m%d") + ".txt" autograding_log_file = os.path.join(AUTOGRADING_LOG_PATH, datefile) easy_to_read_date = dateutils.write_submitty_date(now) my_pid = os.getpid() parent_pid = os.getppid() batch_string = "BATCH" if is_batch else "" abbrev_jobname = jobname[len(SUBMITTY_DATA_DIR + "/courses/"):] time_unit = "" if elapsed_time == "" else "sec" with open(autograding_log_file, 'a') as myfile: fcntl.flock(myfile, fcntl.LOCK_EX | fcntl.LOCK_NB) print("%s | %6s | %5s | %11s | %-75s | %-6s %5s %3s | %s" % (easy_to_read_date, my_pid, batch_string, which_untrusted, abbrev_jobname, timelabel, elapsed_time, time_unit, message), file=myfile) fcntl.flock(myfile, fcntl.LOCK_UN)
def log_stack_trace(job_id="UNKNOWN", is_batch=False, which_untrusted="", jobname="", timelabel="", elapsed_time=-1, trace=""): now = dateutils.get_current_time() datefile = "stack_traces_{0}.txt".format(datetime.strftime(now, "%Y%m%d")) autograding_log_file = os.path.join(AUTOGRADING_STACKTRACE_PATH, datefile) easy_to_read_date = dateutils.write_submitty_date(now, True) batch_string = "BATCH" if is_batch else "" if elapsed_time == "": elapsed_time = -1 elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format(elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" with open(autograding_log_file, 'a') as myfile: try: fcntl.flock(myfile,fcntl.LOCK_EX | fcntl.LOCK_NB) print("%s | %6s | %5s | %11s | %-75s | %-6s %9s %3s |\n%s" % (easy_to_read_date, job_id, batch_string, which_untrusted, jobname, timelabel, elapsed_time_string, time_unit, trace), file=myfile) fcntl.flock(myfile, fcntl.LOCK_UN) except: print("Could not gain a lock on the log file.")
def log_stack_trace( self, trace: str, *, is_batch: bool = False, which_untrusted: str = '', job_id: Optional[str] = None, jobname: str = "", echo_source: Optional[str] = None, ): """Log a stack trace to this logger's configured stack trace directory.""" job_id = job_id or self.job_id # Save the parameters to this trace so we can duplicate these on the # shipper's end once the job finishes. # # TODO: Maybe we want to store time info too? Might need to think a bit # more in terms of the stack traces log file format. if self.capture_traces: self.accumulated_traces.append({ 'trace': trace, 'is_batch': is_batch, 'which_untrusted': which_untrusted, 'job_id': job_id, 'jobname': jobname, }) # Always run this since this could be deleted without us knowing os.makedirs(self.stack_trace_dir, exist_ok=True) now = dateutils.get_current_time() easy_to_read_date = dateutils.write_submitty_date(now, True) message = f"[{easy_to_read_date}][{job_id:>6s}]\n" if echo_source is not None: message += f"== (Echoed from {echo_source})\n" message += f"== Batch? {is_batch}\n" message += f"== Which: {which_untrusted}\n" message += f"== Job: {jobname}\n" for line in trace.splitlines(): message += f"== {line}\n" message = message.strip() write_to_log(self.stack_trace_path, message)
def log_message( self, message: str, *, is_batch: bool = False, which_untrusted: str = "", jobname: str = "", timelabel: str = "", elapsed_time: Optional[int] = None, job_id: Optional[str] = None ): """Log a message to this logger's configured log directory.""" now = dateutils.get_current_time() easy_to_read_date = dateutils.write_submitty_date(now, True) batch_string = "BATCH" if is_batch else "" if elapsed_time is None: elapsed_time = -1 elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format(elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" job_id = job_id or self.job_id parts = (easy_to_read_date, f"{job_id:>6s}", f"{batch_string:>5s}", f"{which_untrusted:>11s}", f"{jobname:75s}", f"{timelabel:6s} {elapsed_time_string:>9s} {time_unit:>3s}", message) write_to_log(self.log_path, ' | '.join((str(x) for x in parts)))
def test_write_submitty_date_microseconds(self, get_timezone): testcases = ( (datetime(2020, 6, 12, 3, 21, 30, tzinfo=pytz_timezone('UTC')), '2020-06-12 03:21:30.000+0000'), (datetime(2020, 6, 12, 3, 21, 30, 123500, tzinfo=pytz_timezone('UTC')), '2020-06-12 03:21:30.123+0000'), (datetime(2020, 6, 12, 3, 21, 30, 211500), '2020-06-12 03:21:30.211-0400'), ) for testcase in testcases: with self.subTest(i=testcase[0]): self.assertEqual( testcase[1], dateutils.write_submitty_date(testcase[0], True))
def log_message(is_batch=False, which_untrusted="", jobname="", timelabel="", elapsed_time=-1, message=""): now = dateutils.get_current_time() datefile = datetime.strftime(now, "%Y%m%d") + ".txt" autograding_log_file = os.path.join(AUTOGRADING_LOG_PATH, datefile) easy_to_read_date = dateutils.write_submitty_date(now, True) my_pid = os.getpid() parent_pid = os.getppid() batch_string = "BATCH" if is_batch else "" elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format( elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" with open(autograding_log_file, 'a') as myfile: fcntl.flock(myfile, fcntl.LOCK_EX | fcntl.LOCK_NB) print("%s | %6s | %5s | %11s | %-75s | %-6s %9s %3s | %s" % (easy_to_read_date, my_pid, batch_string, which_untrusted, jobname, timelabel, elapsed_time_string, time_unit, message), file=myfile) fcntl.flock(myfile, fcntl.LOCK_UN)
def archive_autograding_results(config, working_directory: os.PathLike, job_id: str, which_untrusted: str, is_batch_job: bool, complete_config_obj: dict, gradeable_config_obj: dict, queue_obj: dict, is_test_environment: bool): """ After grading is finished, archive the results. """ tmp_autograding = os.path.join(working_directory, "TMP_AUTOGRADING") tmp_submission = os.path.join(working_directory, "TMP_SUBMISSION") tmp_work = os.path.join(working_directory, "TMP_WORK") tmp_logs = os.path.join(working_directory, "TMP_SUBMISSION", "tmp_logs") tmp_results = os.path.join(working_directory, "TMP_RESULTS") submission_path = os.path.join(tmp_submission, "submission") random_output_path = os.path.join(tmp_work, 'random_output') if "generate_output" not in queue_obj: partial_path = os.path.join(queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"], queue_obj["course"], "submissions", partial_path) elif queue_obj["generate_output"]: item_name = os.path.join(queue_obj["semester"], queue_obj["course"], "generated_output", queue_obj["gradeable"]) results_public_dir = os.path.join(tmp_results, "results_public") results_details_dir = os.path.join(tmp_results, "details") patterns = complete_config_obj['autograding'] # Copy work to details pattern_copy("work_to_details", patterns['work_to_details'], tmp_work, results_details_dir, tmp_logs) # Copy work to public if 'work_to_public' in patterns: pattern_copy("work_to_public", patterns['work_to_public'], tmp_work, results_public_dir, tmp_logs) if os.path.exists(random_output_path): pattern_copy("work_to_random_output", [ os.path.join(random_output_path, '**', '*.txt'), ], tmp_work, tmp_results, tmp_logs) # timestamp of first access to the gradeable page first_access_string = "" # grab the submission time if "generate_output" in queue_obj and queue_obj["generate_output"]: submission_string = "" else: with open( os.path.join(tmp_submission, 'submission', ".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() # grab the first access to the gradeable page (if it exists) user_assignment_access_filename = os.path.join( tmp_submission, ".user_assignment_access.json") if os.path.exists(user_assignment_access_filename): with open(user_assignment_access_filename, 'r') as access_file: obj = json.load(access_file) first_access_string = obj[0]["timestamp"] history_file_tmp = os.path.join(tmp_submission, "history.json") history_file = os.path.join(tmp_results, "history.json") if os.path.isfile(history_file_tmp) and not is_test_environment: shutil.move(history_file_tmp, history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file, int(config.submitty_users['daemon_uid']), ta_group_id) add_permissions(history_file, stat.S_IRGRP) grading_finished = dateutils.get_current_time() grade_result = "" if "generate_output" not in queue_obj: try: shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results) with open(os.path.join(tmp_work, "grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line except Exception as e: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: f.write( f"\n\nERROR: Grading incomplete -- Could not process {os.path.join(tmp_work,'grade.txt')}" ) config.logger.log_message( "ERROR: could not process grade.txt. See stack trace entry for more details.", job_id=job_id, is_batch=is_batch_job, which_untrusted=which_untrusted, jobname=item_name, ) config.logger.log_stack_trace( traceback.format_exc(), job_id=job_id, is_batch=is_batch_job, which_untrusted=which_untrusted, jobname=item_name, ) submission_datetime = dateutils.read_submitty_date(submission_string) submission_longstring = dateutils.write_submitty_date( submission_datetime) # compute lateness (if there is a due date / submission deadline) gradeable_deadline_string = gradeable_config_obj["date_due"] if gradeable_deadline_string is None: print("NO DEADLINE") gradeable_deadline_longstring = "None" seconds_late = 0 else: print("DEADLINE IS '" + str(gradeable_deadline_string) + "'") gradeable_deadline_datetime = dateutils.read_submitty_date( gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date( gradeable_deadline_datetime) seconds_late = int((submission_datetime - gradeable_deadline_datetime).total_seconds()) # compute the access duration in seconds (if it exists) access_duration = -1 if first_access_string != "": first_access_datetime = dateutils.read_submitty_date( first_access_string) access_duration = int( (submission_datetime - first_access_datetime).total_seconds()) # note: negative = not late grading_finished_longstring = dateutils.write_submitty_date( grading_finished) with open(os.path.join(tmp_submission, ".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) gradingtime = (grading_finished - grading_began).total_seconds() queue_obj["gradingtime"] = gradingtime queue_obj["grade_result"] = grade_result queue_obj["which_untrusted"] = which_untrusted waittime = queue_obj["waittime"] try: # Make certain results.json is utf-8 encoded. results_json_path = os.path.join(tmp_work, 'results.json') with codecs.open(results_json_path, 'r', encoding='utf-8', errors='ignore') as infile: results_str = "".join(line.rstrip() for line in infile) results_obj = json.loads(results_str) with open(results_json_path, 'w') as outfile: json.dump(results_obj, outfile, indent=4) shutil.move(results_json_path, os.path.join(tmp_results, "results.json")) except Exception as e: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: f.write( f"\n\nERROR: Grading incomplete -- Could not open/write {os.path.join(tmp_work,'results.json')}" ) config.logger.log_message( "ERROR: results.json read/write error", job_id=job_id, is_batch=is_batch_job, which_untrusted=which_untrusted, jobname=item_name, ) config.logger.log_stack_trace( traceback.format_exc(), job_id=job_id, is_batch=is_batch_job, which_untrusted=which_untrusted, jobname=item_name, ) # Rescue custom validator files custom_validator_output_directory = os.path.join( tmp_results, "custom_validator_output") pattern_copy("rescue_custom_validator_validation_jsons", [ os.path.join(tmp_work, 'validation_results_*.json'), ], tmp_work, custom_validator_output_directory, tmp_logs) pattern_copy("rescue_custom_validator_logs", [ os.path.join(tmp_work, 'validation_logfile_*.txt'), ], tmp_work, custom_validator_output_directory, tmp_logs) pattern_copy("rescue_custom_validator_errors", [ os.path.join(tmp_work, 'validation_stderr_*.txt'), ], tmp_work, custom_validator_output_directory, tmp_logs) just_write_grade_history(history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, first_access_string, access_duration, queue_obj["queue_time"], "BATCH" if is_batch_job else "INTERACTIVE", grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result, queue_obj.get("revision", None)) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: f.write("FINISHED GRADING!\n") config.logger.log_message(grade_result, job_id=job_id, is_batch=is_batch_job, which_untrusted=which_untrusted, jobname=item_name, timelabel="grade:", elapsed_time=gradingtime) with open(os.path.join(tmp_results, "queue_file.json"), 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) # save the logs! shutil.copytree(tmp_logs, os.path.join(tmp_results, "logs")) # Save the .submit.notebook # Copy the .submit.notebook to tmp_work for validation submit_notebook_path = os.path.join(tmp_submission, 'submission', ".submit.notebook") if os.path.exists(submit_notebook_path): shutil.copy(submit_notebook_path, os.path.join(tmp_results, ".submit.notebook"))
def main(): #In order to successfully re-upload the assignment, we need the directory of the assignment to be inflated, #and the semester, course name, and assignment name that it should be uploaded to. parser = argparse.ArgumentParser(description='This script was created to help professors re-upload an old semester\'s\ assignments for autograding and reevaluation. To run it, please create a new assignment via the submitty interface to\ which you wish to upload a past assignment\'s submissions. Then, run this script.') parser.add_argument('-g', '--grade', action='store_true', help='adds assignments to the grading queue.') parser.add_argument('ARCHIVED_directory', help='The path to the top level of the old assignment\'s directory tree. This folder\'s\ subdirectories should mirror a /var/local/submitty/courses/<semester>/<course>/submissions/<assignment_name> folder.') parser.add_argument('semester', help='The semester of the course you wish to upload to.') parser.add_argument('course_name', help='The name of the course you wish to upload to.') parser.add_argument('assignment_name', help='The assignment name you wish to upload to.') args = parser.parse_args() print("You are about to attempt to copy\n\tSOURCE: " + args.ARCHIVED_directory + "\n\tDESTINATION: " + SUBMITTY_DATA_DIR + "/" + args.semester+"/courses/"+args.course_name+"/submissions/"+args.assignment_name) #Check the existence of the submission path (the user should have already created the assignment for us to populate) CURRENT_course_path = os.path.join(SUBMITTY_DATA_DIR, "courses", args.semester, args.course_name) print("Current course path:" + CURRENT_course_path) CURRENT_assignment_path = os.path.join(CURRENT_course_path, "submissions", args.assignment_name) print("Current assignment path" + CURRENT_assignment_path) #we check the bin directory to see if the assignment has ever been built. (Not a guarantee, but high probability of success) CURRENT_bin_directory = os.path.join(CURRENT_course_path, "bin", args.assignment_name) if not os.path.isdir(CURRENT_bin_directory): raise SystemExit("ERROR: The directory " + CURRENT_bin_directory + " does not exist. Please make sure that you\n\t\ 1) Configured the assignment on the course website\n\t2) Did not mistype the program arguments.") else: print("SUCCESS: The directory " + CURRENT_assignment_path + " does exist.") #Make a connection to the database and grab the necessary tables. database = "submitty_" + args.semester + "_" + args.course_name print("Connecting to database: ", end="") engine = create_engine("postgresql://{}:{}@{}/{}".format(DB_USER, DB_PASS, DB_HOST, database)) conn = engine.connect() metadata = MetaData(bind=engine) print("(connection made, metadata bound)...") electronic_gradeable_data = Table("electronic_gradeable_data", metadata, autoload=True) electronic_gradeable_version = Table("electronic_gradeable_version", metadata, autoload=True) course_group = grp.getgrgid(os.stat(os.path.join(SUBMITTY_DATA_DIR,"courses",args.semester,args.course_name)).st_gid)[0] #For every user folder in our directory, for every submission folder inside of it, copy over that user/submission, add it to #the database, and create a queue file. for user_folder in sorted(os.listdir(args.ARCHIVED_directory)): print("evaluating " + user_folder) #If the item we are currently looking at is a directory, we will assume it is a student submission directory. ARCHIVED_user_dir = os.path.join(args.ARCHIVED_directory, user_folder) print("Starting work on archived dir: " + ARCHIVED_user_dir) if not os.path.isdir(ARCHIVED_user_dir): print("Skipping the following as it is not a directory: " + ARCHIVED_user_dir) continue user_name = user_folder print("processing user: "******"user_assignment_settings.json"), "r") as open_file: user_assignment_settings = json.load(open_file) #For every folder inside of the user submission folder. for submission in os.listdir(ARCHIVED_user_dir): ARCHIVED_submission_path = os.path.join(ARCHIVED_user_dir, submission) print("Evaluating submission: " + ARCHIVED_submission_path) #If this entry in the directory to be copied is not a directory, it is a user_assignments settings. #Right now, we ignore these. #TODO copy over the user_assignment_settings files and use them instead of creating new ones. if not os.path.isdir(ARCHIVED_submission_path): if user_folder == ".submit.timestamp": with open(ARCHIVED_submission_path, 'r') as old_timestamp, open(CURRENT_user_path, 'w') as new_timestamp: for line in old_timestamp: new_timestamp.write(line) os.system("chown -R submitty_php:{} {}".format(course_group, new_timestamp)) continue #if the student's submission dir does not exist, make it. if not os.path.isdir(CURRENT_user_path): os.makedirs(CURRENT_user_path) #The current directory for the new submission CURRENT_submission_path = os.path.join(CURRENT_user_path, submission) #TODO If the submission already exists, give up? if os.path.isdir(CURRENT_submission_path): print("Skipped " + CURRENT_submission_path + " as it already exists.") continue #This permission also sets the underlying submission paths recursively.g print("Set permissions on the submission") os.system("chown -R submitty_php:{} {}".format(course_group, CURRENT_user_path)) #TODO: Sort the submissions so that they are guaranteed to be given in chronological (1,2,3,etc) order. #copy in the submission directory. print("Copied from\n\tSOURCE: " + ARCHIVED_submission_path +"\n\t" + "DESTINATION: " + CURRENT_submission_path) shutil.copytree(ARCHIVED_submission_path, CURRENT_submission_path) #give the appropriate permissions os.system("chown -R submitty_php:{} {}".format(course_group, CURRENT_submission_path)) #add each submission to the database. current_time_string = dateutils.write_submitty_date() conn.execute(electronic_gradeable_data.insert(), g_id=args.assignment_name, user_id=user_name, g_version=submission, submission_time=current_time_string) #If this is the first submission, create a new entry in the table, otherwise, update. #TODO use a more reliable method of determining if this is the first submission. if int(submission) == 1: print("Entered new user " + user_name + " because submission was " + submission) conn.execute(electronic_gradeable_version.insert(), g_id=args.assignment_name, user_id=user_name, active_version=user_assignment_settings['active_version']) else: print("UPDATED: where g_id is " + args.assignment_name + " and user id is " + user_name + " to value " + str(user_assignment_settings['active_version'])) stmt = electronic_gradeable_version.update().\ where(and_(electronic_gradeable_version.c.g_id==args.assignment_name, electronic_gradeable_version.c.user_id==user_name)).\ values(active_version=user_assignment_settings['active_version']) conn.execute(stmt) with open(os.path.join(CURRENT_user_path, "user_assignment_settings.json"), "w") as open_file: json.dump(user_assignment_settings, open_file, indent = 4) if args.grade: # Create a queue file for each submission queue_file = "__".join([args.semester, args.course_name, args.assignment_name, user_name, submission]) print("Creating queue file:", queue_file) queue_file = os.path.join(SUBMITTY_DATA_DIR, "to_be_graded_queue", queue_file) with open(queue_file, "w") as open_file: # FIXME: This will need to be adjusted for team assignments # and assignments with special required capabilities! queue_time = dateutils.write_submitty_date() json.dump({"semester": args.semester, "course": args.course_name, "gradeable": args.assignment_name, "user": user_name, "team": "", "who": user_name, "is_team": False, "version": submission, "required_capabilities" : "default", "queue_time": queue_time, "regrade": True, "max_possible_grading_time": -1}, open_file) conn.close()
def prepare_autograding_and_submission_zip(which_machine, which_untrusted, next_directory, next_to_grade): os.chdir(SUBMITTY_DATA_DIR) # generate a random id to be used to track this job in the autograding logs job_id = ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(6)) # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = load_queue_file_obj(job_id, next_directory, next_to_grade) partial_path = os.path.join(obj["gradeable"], obj["who"], str(obj["version"])) item_name = os.path.join(obj["semester"], obj["course"], "submissions", partial_path) submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name) if not os.path.isdir(submission_path): grade_items_logging.log_message( job_id, message="ERROR: the submission directory does not exist " + submission_path) raise RuntimeError("ERROR: the submission directory does not exist", submission_path) print(which_machine, which_untrusted, "prepare zip", submission_path) is_vcs, vcs_type, vcs_base_url, vcs_subdirectory = get_vcs_info( SUBMITTY_DATA_DIR, obj["semester"], obj["course"], obj["gradeable"], obj["who"], obj["team"]) is_batch_job = "regrade" in obj and obj["regrade"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory, next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = (grading_began - queue_time).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, "zip", item_name, "wait:", waittime, "") # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "provided_code", obj["gradeable"]) instructor_solution_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "instructor_solution", obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_input", obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_output", obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "custom_validation_code", obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "bin", obj["gradeable"]) form_json_config = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "form", "form_" + obj["gradeable"] + ".json") complete_config = os.path.join( SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "complete_config", "complete_config_" + obj["gradeable"] + ".json") if not os.path.exists(form_json_config): grade_items_logging.log_message( job_id, message="ERROR: the form json file does not exist " + form_json_config) raise RuntimeError("ERROR: the form json file does not exist ", form_json_config) if not os.path.exists(complete_config): grade_items_logging.log_message( job_id, message="ERROR: the complete config file does not exist " + complete_config) raise RuntimeError("ERROR: the complete config file does not exist ", complete_config) # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = tempfile.mkdtemp() tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") os.mkdir(tmp_autograding) tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") os.mkdir(tmp_submission) copytree_if_exists(provided_code_path, os.path.join(tmp_autograding, "provided_code")) copytree_if_exists(instructor_solution_path, os.path.join(tmp_autograding, "instructor_solution")) copytree_if_exists(test_input_path, os.path.join(tmp_autograding, "test_input")) copytree_if_exists(test_output_path, os.path.join(tmp_autograding, "test_output")) copytree_if_exists(custom_validation_code_path, os.path.join(tmp_autograding, "custom_validation_code")) copytree_if_exists(bin_path, os.path.join(tmp_autograding, "bin")) shutil.copy(form_json_config, os.path.join(tmp_autograding, "form.json")) shutil.copy(complete_config, os.path.join(tmp_autograding, "complete_config.json")) checkout_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "checkout", partial_path) results_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "results", partial_path) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path, "history.json") history_file_tmp = "" if os.path.isfile(history_file): shutil.copy(history_file, os.path.join(tmp_submission, "history.json")) # get info from the gradeable config file with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) queue_file = os.path.join(next_directory, next_to_grade) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") os.makedirs(tmp_logs) # 'touch' a file in the logs folder open(os.path.join(tmp_logs, "overall.txt"), 'a') # -------------------------------------------------------------------- # CONFIRM WE HAVE A CHECKOUT OF THE STUDENT'S REPO if is_vcs: # there should be a checkout log file in the results directory # move that file to the tmp logs directory.. vcs_checkout_logfile = os.path.join(results_path, "logs", "vcs_checkout.txt") if os.path.isfile(vcs_checkout_logfile): shutil.move(vcs_checkout_logfile, tmp_logs) else: grade_items_logging.log_message( JOB_ID, message=str(my_name) + " ERROR: missing vcs_checkout.txt logfile " + str(vcs_checkout_logfile)) copytree_if_exists(submission_path, os.path.join(tmp_submission, "submission")) copytree_if_exists(checkout_path, os.path.join(tmp_submission, "checkout")) obj["queue_time"] = queue_time_longstring obj["regrade"] = is_batch_job obj["waittime"] = waittime obj["job_id"] = job_id with open(os.path.join(tmp_submission, "queue_file.json"), 'w') as outfile: json.dump(obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) grading_began_longstring = dateutils.write_submitty_date(grading_began) with open(os.path.join(tmp_submission, ".grading_began"), 'w') as f: print(grading_began_longstring, file=f) # zip up autograding & submission folders filehandle1, my_autograding_zip_file = tempfile.mkstemp() filehandle2, my_submission_zip_file = tempfile.mkstemp() grade_item.zip_my_directory(tmp_autograding, my_autograding_zip_file) grade_item.zip_my_directory(tmp_submission, my_submission_zip_file) os.close(filehandle1) os.close(filehandle2) # cleanup shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp) #grade_items_logging.log_message(job_id,is_batch_job,"done zip",item_name) return (my_autograding_zip_file, my_submission_zip_file)
def replay(starttime,endtime): replay_starttime=datetime.datetime.now() print (replay_starttime,"replay start: ",starttime) # error checking if not (starttime.year == endtime.year and starttime.month == endtime.month and starttime.day == endtime.day): print ("ERROR! invalid replay range ",starttime,"->",endtime, " (must be same day)") exit() if starttime >= endtime: print ("ERROR! invalid replay range ",starttime,"->",endtime, " (invalid times)") exit() # file the correct file file = '/var/local/submitty/logs/autograding/{:d}{:02d}{:02d}.txt'.format(starttime.year,starttime.month,starttime.day) with open(file,'r') as lines: for line in lines: things = line.split('|') original_time = dateutils.read_submitty_date(things[0]) # skip items outside of this time range if (original_time < starttime or original_time > endtime): continue # skip batch items if (things[2].strip() == "BATCH"): continue # only process the "wait" time (when we started grading the item) iswait=things[5].strip()[0:5] if (iswait != "wait:"): continue waittime=float(things[5].split()[1]) # grab the job name my_job = things[4].strip() if my_job == "": continue what = my_job.split('/') # for now, only interested in Data Structures and Computer Science 1 if not (what[1]=="csci1200" or what[1]=="csci1100"): continue # calculate when this job should be relaunched time_multipler=1.0 pause_time=replay_starttime+(time_multiplier*(original_time-starttime)) pause.until(pause_time) queue_time = dateutils.write_submitty_date() print(datetime.datetime.now()," REPLAY: ",original_time," ",my_job) # FIXME : This will need to be adjust for team assigments # and assignments with special required capabilities! item = {"semester": what[0], "course": what[1], "gradeable": what[3], "user": what[4], "team": "", "who": what[4], "is_team": False, "version": what[5], "required_capabilities": "default", "queue_time": queue_time, "regrade": True, "max_possible_grading_time" : -1 } file_name = "__".join([item['semester'], item['course'], item['gradeable'], item['who'], item['version']]) file_name = os.path.join(SUBMITTY_DATA_DIR, "to_be_graded_queue", file_name) with open(file_name, "w") as open_file: json.dump(item, open_file, sort_keys=True, indent=4) os.system("chmod o+rw {}".format(file_name)) print (datetime.datetime.now(),"replay end: ",endtime)
def grade_from_zip(my_autograding_zip_file, my_submission_zip_file, which_untrusted): os.chdir(SUBMITTY_DATA_DIR) tmp = os.path.join("/var/local/submitty/autograding_tmp/", which_untrusted, "tmp") if os.path.exists(tmp): untrusted_grant_rwx_access(which_untrusted, tmp) add_permissions_recursive( tmp, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # Remove any and all containers left over from past runs. old_containers = subprocess.check_output( ['docker', 'ps', '-aq', '-f', 'name={0}'.format(which_untrusted)]).split() for old_container in old_containers: subprocess.call(['docker', 'rm', '-f', old_container.decode('utf8')]) # clean up old usage of this directory shutil.rmtree(tmp, ignore_errors=True) os.mkdir(tmp) which_machine = socket.gethostname() # unzip autograding and submission folders tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") try: unzip_this_file(my_autograding_zip_file, tmp_autograding) unzip_this_file(my_submission_zip_file, tmp_submission) except: raise os.remove(my_autograding_zip_file) os.remove(my_submission_zip_file) tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") queue_file = os.path.join(tmp_submission, "queue_file.json") with open(queue_file, 'r') as infile: queue_obj = json.load(infile) queue_time_longstring = queue_obj["queue_time"] waittime = queue_obj["waittime"] is_batch_job = queue_obj["regrade"] job_id = queue_obj["job_id"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" revision = queue_obj.get("revision", None) partial_path = os.path.join(queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"], queue_obj["course"], "submissions", partial_path) grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "wait:", waittime, "") with open(os.path.join(tmp_submission, ".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) submission_path = os.path.join(tmp_submission, "submission") checkout_path = os.path.join(tmp_submission, "checkout") provided_code_path = os.path.join(tmp_autograding, "provided_code") test_input_path = os.path.join(tmp_autograding, "test_input") test_output_path = os.path.join(tmp_autograding, "test_output") custom_validation_code_path = os.path.join(tmp_autograding, "custom_validation_code") bin_path = os.path.join(tmp_autograding, "bin") form_json_config = os.path.join(tmp_autograding, "form.json") complete_config = os.path.join(tmp_autograding, "complete_config.json") with open(form_json_config, 'r') as infile: gradeable_config_obj = json.load(infile) gradeable_deadline_string = gradeable_config_obj["date_due"] with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) is_vcs = gradeable_config_obj["upload_type"] == "repository" checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) if complete_config_obj.get('one_part_only', False): allow_only_one_part(submission_path, os.path.join(tmp_logs, "overall.txt")) if is_vcs: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print( "WARNING: ONE_PART_ONLY OPTION DOES NOT MAKE SENSE WITH VCS SUBMISSION", file=f) # -------------------------------------------------------------------- # START DOCKER # NOTE: DOCKER SUPPORT PRELIMINARY -- NEEDS MORE SECURITY BEFORE DEPLOYED ON LIVE SERVER complete_config = os.path.join(tmp_autograding, "complete_config.json") with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) # Save ourselves if autograding_method is None. autograding_method = complete_config_obj.get("autograding_method", "") USE_DOCKER = True if autograding_method == "docker" else False # -------------------------------------------------------------------- # COMPILE THE SUBMITTED CODE with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nCOMPILATION STARTS", file=f) # copy submitted files to the tmp compilation directory tmp_compilation = os.path.join(tmp, "TMP_COMPILATION") os.mkdir(tmp_compilation) os.chdir(tmp_compilation) submission_path = os.path.join(tmp_submission, "submission") checkout_path = os.path.join(tmp_submission, "checkout") provided_code_path = os.path.join(tmp_autograding, "provided_code") test_input_path = os.path.join(tmp_autograding, "test_input") test_output_path = os.path.join(tmp_autograding, "test_output") custom_validation_code_path = os.path.join(tmp_autograding, "custom_validation_code") bin_path = os.path.join(tmp_autograding, "bin") form_json_config = os.path.join(tmp_autograding, "form.json") with open(form_json_config, 'r') as infile: gradeable_config_obj = json.load(infile) gradeable_deadline_string = gradeable_config_obj["date_due"] patterns_submission_to_compilation = complete_config_obj["autograding"][ "submission_to_compilation"] add_permissions(tmp_logs, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) if USE_DOCKER: print("!!!!!!!!!!!!!!!!!!USING DOCKER!!!!!!!!!!!!!!!!!!!!!!!!") with open(complete_config, 'r') as infile: config = json.load(infile) my_testcases = config['testcases'] # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() with open(os.path.join(tmp_logs, "compilation_log.txt"), 'w') as logfile: # we start counting from one. executable_path_list = list() for testcase_num in range(1, len(my_testcases) + 1): testcase_folder = os.path.join(tmp_compilation, "test{:02}".format(testcase_num)) if 'type' in my_testcases[testcase_num - 1]: if my_testcases[testcase_num - 1]['type'] != 'FileCheck' and my_testcases[ testcase_num - 1]['type'] != 'Compilation': continue if my_testcases[testcase_num - 1]['type'] == 'Compilation': if 'executable_name' in my_testcases[testcase_num - 1]: provided_executable_list = my_testcases[ testcase_num - 1]['executable_name'] if not isinstance(provided_executable_list, (list, )): provided_executable_list = list( [provided_executable_list]) for executable_name in provided_executable_list: if executable_name.strip() == '': continue executable_path = os.path.join( testcase_folder, executable_name) executable_path_list.append( (executable_path, executable_name)) else: continue os.makedirs(testcase_folder) pattern_copy("submission_to_compilation", patterns_submission_to_compilation, submission_path, testcase_folder, tmp_logs) if is_vcs: pattern_copy("checkout_to_compilation", patterns_submission_to_compilation, checkout_subdir_path, testcase_folder, tmp_logs) # copy any instructor provided code files to tmp compilation directory copy_contents_into(job_id, provided_code_path, testcase_folder, tmp_logs) # copy compile.out to the current directory shutil.copy(os.path.join(bin_path, "compile.out"), os.path.join(testcase_folder, "my_compile.out")) add_permissions( os.path.join(testcase_folder, "my_compile.out"), stat.S_IXUSR | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #untrusted_grant_rwx_access(which_untrusted, tmp_compilation) untrusted_grant_rwx_access(which_untrusted, testcase_folder) add_permissions_recursive( testcase_folder, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) if USE_DOCKER: try: #There can be only one container for a compilation step, so grab its container image #TODO: set default in load_config_json.cpp if my_testcases[testcase_num - 1]['type'] == 'FileCheck': print( "performing filecheck in default ubuntu:custom container" ) container_image = "ubuntu:custom" else: container_image = my_testcases[ testcase_num - 1]["containers"][0]["container_image"] print( 'creating a compilation container with image {0}'. format(container_image)) untrusted_uid = str(getpwnam(which_untrusted).pw_uid) compilation_container = None compilation_container = subprocess.check_output([ 'docker', 'create', '-i', '-u', untrusted_uid, '--network', 'none', '-v', testcase_folder + ':' + testcase_folder, '-w', testcase_folder, container_image, #The command to be run. os.path.join(testcase_folder, 'my_compile.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string, '--testcase', str(testcase_num) ]).decode('utf8').strip() print("starting container") compile_success = subprocess.call( ['docker', 'start', '-i', compilation_container], stdout=logfile, cwd=testcase_folder) except Exception as e: print('An error occurred when compiling with docker.') grade_items_logging.log_stack_trace( job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) finally: if compilation_container != None: subprocess.call( ['docker', 'rm', '-f', compilation_container]) print("cleaned up compilation container.") else: compile_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(testcase_folder, "my_compile.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string, '--testcase', str(testcase_num) ], stdout=logfile, cwd=testcase_folder) # remove the compilation program untrusted_grant_rwx_access(which_untrusted, testcase_folder) os.remove(os.path.join(testcase_folder, "my_compile.out")) if compile_success == 0: print(which_machine, which_untrusted, "COMPILATION OK") else: print(which_machine, which_untrusted, "COMPILATION FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="COMPILATION FAILURE") add_permissions_recursive( tmp_compilation, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # return to the main tmp directory os.chdir(tmp) # -------------------------------------------------------------------- # make the runner directory with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nRUNNER STARTS", file=f) tmp_work = os.path.join(tmp, "TMP_WORK") tmp_work_test_input = os.path.join(tmp_work, "test_input") tmp_work_submission = os.path.join(tmp_work, "submitted_files") tmp_work_compiled = os.path.join(tmp_work, "compiled_files") tmp_work_checkout = os.path.join(tmp_work, "checkout") os.mkdir(tmp_work) os.mkdir(tmp_work_test_input) os.mkdir(tmp_work_submission) os.mkdir(tmp_work_compiled) os.mkdir(tmp_work_checkout) os.chdir(tmp_work) # move all executable files from the compilation directory to the main tmp directory # Note: Must preserve the directory structure of compiled files (esp for Java) patterns_submission_to_runner = complete_config_obj["autograding"][ "submission_to_runner"] pattern_copy("submission_to_runner", patterns_submission_to_runner, submission_path, tmp_work_submission, tmp_logs) if is_vcs: pattern_copy("checkout_to_runner", patterns_submission_to_runner, checkout_subdir_path, tmp_work_checkout, tmp_logs) # move the compiled files into the tmp_work_compiled directory for path, name in executable_path_list: if not os.path.isfile(path): continue target_path = os.path.join(tmp_work_compiled, name) if not os.path.exists(target_path): os.makedirs(os.path.dirname(target_path), exist_ok=True) shutil.copy(path, target_path) print('copied over {0}'.format(target_path)) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print('grade_item: copied over {0}'.format(target_path), file=f) patterns_compilation_to_runner = complete_config_obj["autograding"][ "compilation_to_runner"] #copy into the actual tmp_work directory for archiving/validating pattern_copy("compilation_to_runner", patterns_compilation_to_runner, tmp_compilation, tmp_work, tmp_logs) #copy into tmp_work_compiled, which is provided to each testcase # TODO change this as our methodology for declaring testcase dependencies becomes more robust pattern_copy("compilation_to_runner", patterns_compilation_to_runner, tmp_compilation, tmp_work_compiled, tmp_logs) # copy input files to tmp_work directory copy_contents_into(job_id, test_input_path, tmp_work_test_input, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy runner.out to the current directory shutil.copy(os.path.join(bin_path, "run.out"), os.path.join(tmp_work, "my_runner.out")) #set the appropriate permissions for the newly created directories #TODO replaces commented out code below add_permissions( os.path.join(tmp_work, "my_runner.out"), stat.S_IXUSR | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_submission, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_compiled, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_checkout, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #TODO this is how permissions used to be set. It was removed because of the way it interacts with the sticky bit. ## give the untrusted user read/write/execute permissions on the tmp directory & files # os.system('ls -al {0}'.format(tmp_work)) # add_permissions_recursive(tmp_work, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) ################################################################################################## #call grade_item_main_runner.py runner_success = grade_item_main_runner.executeTestcases( complete_config_obj, tmp_logs, tmp_work, queue_obj, submission_string, item_name, USE_DOCKER, None, which_untrusted, job_id, grading_began) ################################################################################################## if runner_success == 0: print(which_machine, which_untrusted, "RUNNER OK") else: print(which_machine, which_untrusted, "RUNNER FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="RUNNER FAILURE") add_permissions_recursive( tmp_work, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions_recursive( tmp_compilation, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # -------------------------------------------------------------------- # RUN VALIDATOR with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nVALIDATION STARTS", file=f) # copy results files from compilation... patterns_submission_to_validation = complete_config_obj["autograding"][ "submission_to_validation"] pattern_copy("submission_to_validation", patterns_submission_to_validation, submission_path, tmp_work, tmp_logs) if is_vcs: pattern_copy("checkout_to_validation", patterns_submission_to_validation, checkout_subdir_path, tmp_work, tmp_logs) patterns_compilation_to_validation = complete_config_obj["autograding"][ "compilation_to_validation"] pattern_copy("compilation_to_validation", patterns_compilation_to_validation, tmp_compilation, tmp_work, tmp_logs) # remove the compilation directory shutil.rmtree(tmp_compilation) # copy output files to tmp_work directory copy_contents_into(job_id, test_output_path, tmp_work, tmp_logs) # copy any instructor custom validation code into the tmp work directory copy_contents_into(job_id, custom_validation_code_path, tmp_work, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy validator.out to the current directory shutil.copy(os.path.join(bin_path, "validate.out"), os.path.join(tmp_work, "my_validator.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions( os.path.join(tmp_work, "my_validator.out"), stat.S_IXUSR | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #todo remove prints. print("VALIDATING") # validator the validator.out as the untrusted user with open(os.path.join(tmp_logs, "validator_log.txt"), 'w') as logfile: if USE_DOCKER: # WIP: This option file facilitated testing... #USE_DOCKER = os.path.isfile("/tmp/use_docker") #use_docker_string="grading begins, using DOCKER" if USE_DOCKER else "grading begins (not using docker)" #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message=use_docker_string) container = subprocess.check_output([ 'docker', 'run', '-t', '-d', '-v', tmp + ':' + tmp, 'ubuntu:custom' ]).decode('utf8').strip() dockerlaunch_done = dateutils.get_current_time() dockerlaunch_time = (dockerlaunch_done - grading_began).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "dcct:", dockerlaunch_time, "docker container created") validator_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_validator.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string ], stdout=logfile) else: validator_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(tmp_work, "my_validator.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string ], stdout=logfile) if validator_success == 0: print(which_machine, which_untrusted, "VALIDATOR OK") else: print(which_machine, which_untrusted, "VALIDATOR FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="VALIDATION FAILURE") untrusted_grant_rwx_access(which_untrusted, tmp_work) # grab the result of autograding grade_result = "" try: with open(os.path.join(tmp_work, "grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line except: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("\n\nERROR: Grading incomplete -- Could not open ", os.path.join(tmp_work, "grade.txt")) grade_items_logging.log_message( job_id, is_batch_job, which_untrusted, item_name, message="ERROR: grade.txt does not exist") grade_items_logging.log_stack_trace(job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) # -------------------------------------------------------------------- # MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE tmp_results = os.path.join(tmp, "TMP_RESULTS") with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nARCHIVING STARTS", file=f) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) os.makedirs(os.path.join(tmp_results, "details")) # remove the test_input directory, so we don't archive it! shutil.rmtree(os.path.join(tmp_work, "test_input")) # loop over the test case directories, and remove any files that are also in the test_input folder for testcase_num in range(1, len(my_testcases) + 1): testcase_folder = os.path.join(tmp_work, "test{:02}".format(testcase_num)) remove_test_input_files(os.path.join(tmp_logs, "overall.txt"), test_input_path, testcase_folder) patterns_work_to_details = complete_config_obj["autograding"][ "work_to_details"] pattern_copy("work_to_details", patterns_work_to_details, tmp_work, os.path.join(tmp_results, "details"), tmp_logs) if ("work_to_public" in complete_config_obj["autograding"] and len(complete_config_obj["autograding"]["work_to_public"]) > 0): # create the directory os.makedirs(os.path.join(tmp_results, "results_public")) # copy the files patterns_work_to_public = complete_config_obj["autograding"][ "work_to_public"] pattern_copy("work_to_public", patterns_work_to_public, tmp_work, os.path.join(tmp_results, "results_public"), tmp_logs) history_file_tmp = os.path.join(tmp_submission, "history.json") history_file = os.path.join(tmp_results, "history.json") if os.path.isfile(history_file_tmp): shutil.move(history_file_tmp, history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file, int(DAEMON_UID), ta_group_id) add_permissions(history_file, stat.S_IRGRP) grading_finished = dateutils.get_current_time() try: shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results) except: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("\n\nERROR: Grading incomplete -- Could not copy ", os.path.join(tmp_work, "grade.txt")) grade_items_logging.log_message( job_id, is_batch_job, which_untrusted, item_name, message="ERROR: grade.txt does not exist") grade_items_logging.log_stack_trace(job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) # ------------------------------------------------------------- # create/append to the results history # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) gradeable_deadline_datetime = dateutils.read_submitty_date( gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date( gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int( (submission_datetime - gradeable_deadline_datetime).total_seconds()) # note: negative = not late grading_finished_longstring = dateutils.write_submitty_date( grading_finished) gradingtime = (grading_finished - grading_began).total_seconds() with open(os.path.join(tmp_submission, "queue_file.json"), 'r') as infile: queue_obj = json.load(infile) queue_obj["gradingtime"] = gradingtime queue_obj["grade_result"] = grade_result queue_obj["which_untrusted"] = which_untrusted with open(os.path.join(tmp_results, "queue_file.json"), 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) try: shutil.move(os.path.join(tmp_work, "results.json"), os.path.join(tmp_results, "results.json")) except: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("\n\nERROR: Grading incomplete -- Could not open/write ", os.path.join(tmp_work, "results.json")) grade_items_logging.log_message( job_id, is_batch_job, which_untrusted, item_name, message="ERROR: results.json read/write error") grade_items_logging.log_stack_trace(job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) write_grade_history.just_write_grade_history( history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_time_longstring, is_batch_job_string, grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result, revision) os.chdir(SUBMITTY_DATA_DIR) if USE_DOCKER: with open(os.path.join(tmp_logs, "overall_log.txt"), 'w') as logfile: chmod_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, 'chmod', '-R', 'ugo+rwx', '.' ], stdout=logfile) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: f.write("FINISHED GRADING!\n") # save the logs! shutil.copytree(tmp_logs, os.path.join(tmp_results, "logs")) # zip up results folder filehandle, my_results_zip_file = tempfile.mkstemp() zip_my_directory(tmp_results, my_results_zip_file) os.close(filehandle) shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp_results) shutil.rmtree(tmp_work) shutil.rmtree(tmp) # WIP: extra logging for testing #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message="done grading") # -------------------------------------------------------------------- # CLEAN UP DOCKER if USE_DOCKER: subprocess.call(['docker', 'rm', '-f', container]) dockerdestroy_done = dateutils.get_current_time() dockerdestroy_time = (dockerdestroy_done - grading_finished).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "ddt:", dockerdestroy_time, "docker container destroyed") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "grade:", gradingtime, grade_result) return my_results_zip_file
def archive_autograding_results(working_directory, job_id, which_untrusted, is_batch_job, complete_config_obj, gradeable_config_obj, queue_obj, log_path, stack_trace_log_path, is_test_environment): """ After grading is finished, archive the results. """ tmp_autograding = os.path.join(working_directory,"TMP_AUTOGRADING") tmp_submission = os.path.join(working_directory,"TMP_SUBMISSION") tmp_work = os.path.join(working_directory,"TMP_WORK") tmp_logs = os.path.join(working_directory,"TMP_SUBMISSION","tmp_logs") tmp_results = os.path.join(working_directory,"TMP_RESULTS") submission_path = os.path.join(tmp_submission, "submission") random_output_path = os.path.join(tmp_work, 'random_output') if "generate_output" not in queue_obj: partial_path = os.path.join(queue_obj["gradeable"],queue_obj["who"],str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"],queue_obj["course"],"submissions",partial_path) elif queue_obj["generate_output"]: item_name = os.path.join(queue_obj["semester"],queue_obj["course"],"generated_output",queue_obj["gradeable"]) results_public_dir = os.path.join(tmp_results,"results_public") results_details_dir = os.path.join(tmp_results, "details") patterns = complete_config_obj['autograding'] # Copy work to details pattern_copy("work_to_details", patterns['work_to_details'], tmp_work, results_details_dir, tmp_logs) # Copy work to public if 'work_to_public' in patterns: pattern_copy("work_to_public", patterns['work_to_public'], tmp_work, results_public_dir, tmp_logs) if os.path.exists(random_output_path): pattern_copy("work_to_random_output", [os.path.join(random_output_path, 'test*', '**', '*.txt'),], tmp_work, tmp_results, tmp_logs) # timestamp of first access to the gradeable page first_access_string = "" # grab the submission time if "generate_output" in queue_obj and queue_obj["generate_output"]: submission_string = "" else: with open(os.path.join(tmp_submission, 'submission' ,".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() # grab the first access to the gradeable page (if it exists) user_assignment_access_filename = os.path.join(tmp_submission, "user_assignment_access.json") if os.path.exists(user_assignment_access_filename): with open(user_assignment_access_filename, 'r') as access_file: obj = json.load(access_file, object_pairs_hook=collections.OrderedDict) first_access_string = obj["page_load_history"][0]["time"] history_file_tmp = os.path.join(tmp_submission,"history.json") history_file = os.path.join(tmp_results,"history.json") if os.path.isfile(history_file_tmp) and not is_test_environment: from . import CONFIG_PATH with open(os.path.join(CONFIG_PATH, 'submitty_users.json')) as open_file: OPEN_JSON = json.load(open_file) DAEMON_UID = OPEN_JSON['daemon_uid'] shutil.move(history_file_tmp, history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file, int(DAEMON_UID),ta_group_id) add_permissions(history_file, stat.S_IRGRP) grading_finished = dateutils.get_current_time() if "generate_output" not in queue_obj: try: shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not copy ",os.path.join(tmp_work,"grade.txt")) log_message(log_path, job_id, is_batch_job, which_untrusted, item_name, message="ERROR: grade.txt does not exist") log_stack_trace(stack_trace_log_path, job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) grade_result = "" try: with open(os.path.join(tmp_work,"grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open ",os.path.join(tmp_work,"grade.txt")) log_message(job_id,is_batch_job,which_untrusted,item_name,message="ERROR: grade.txt does not exist") log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) gradeable_deadline_string = gradeable_config_obj["date_due"] # FIXME: The access date string is currently misformatted # mm-dd-yyyy, but we want yyyy-mm-dd. Also it is missing # the common name timezone string, e.g., "America/NewYork". # We should standardize this logging eventually, but # keeping it as is because we are mid-semester with this # new feature and I don't want to break things. first_access_string = dateutils.normalize_submitty_date(first_access_string) submission_datetime = dateutils.read_submitty_date(submission_string) gradeable_deadline_datetime = dateutils.read_submitty_date(gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date(gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int((submission_datetime-gradeable_deadline_datetime).total_seconds()) # compute the access duration in seconds (if it exists) access_duration = -1 if first_access_string != "": first_access_datetime = dateutils.read_submitty_date(first_access_string) access_duration = int((submission_datetime-first_access_datetime).total_seconds()) # note: negative = not late grading_finished_longstring = dateutils.write_submitty_date(grading_finished) with open(os.path.join(tmp_submission,".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) gradingtime = (grading_finished - grading_began).total_seconds() queue_obj["gradingtime"]=gradingtime queue_obj["grade_result"]=grade_result queue_obj["which_untrusted"]=which_untrusted waittime = queue_obj["waittime"] try: # Make certain results.json is utf-8 encoded. results_json_path = os.path.join(tmp_work, 'results.json') with codecs.open(results_json_path, 'r', encoding='utf-8', errors='ignore') as infile: results_str = "".join(line.rstrip() for line in infile) results_obj = json.loads(results_str) with open(results_json_path, 'w') as outfile: json.dump(results_obj, outfile, indent=4) shutil.move(results_json_path, os.path.join(tmp_results, "results.json")) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open/write ",os.path.join(tmp_work,"results.json")) log_message(log_path, job_id,is_batch_job,which_untrusted,item_name,message="ERROR: results.json read/write error") log_stack_trace(stack_trace_log_path, job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) # Rescue custom validator files custom_validator_output_directory = os.path.join(tmp_results, "custom_validator_output") pattern_copy("rescue_custom_validator_validation_jsons", [os.path.join(tmp_work, 'validation_results_*.json'),], tmp_work, custom_validator_output_directory, tmp_logs) pattern_copy("rescue_custom_validator_logs", [os.path.join(tmp_work, 'validation_logfile_*.txt'),], tmp_work, custom_validator_output_directory, tmp_logs) pattern_copy("rescue_custom_validator_errors", [os.path.join(tmp_work, 'validation_stderr_*.txt'),], tmp_work, custom_validator_output_directory, tmp_logs) just_write_grade_history(history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, first_access_string, access_duration, queue_obj["queue_time"], "BATCH" if is_batch_job else "INTERACTIVE", grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result, queue_obj.get("revision", None)) with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: f.write("FINISHED GRADING!\n") log_message(log_path, job_id,is_batch_job,which_untrusted,item_name,"grade:",gradingtime,grade_result) with open(os.path.join(tmp_results,"queue_file.json"),'w') as outfile: json.dump(queue_obj,outfile,sort_keys=True,indent=4,separators=(',', ': ')) # save the logs! shutil.copytree(tmp_logs,os.path.join(tmp_results,"logs"))
def prepare_job(my_name,which_machine,which_untrusted,next_directory,next_to_grade): # verify the DAEMON_USER is running this script if not int(os.getuid()) == int(DAEMON_UID): grade_items_logging.log_message(JOB_ID, message="ERROR: must be run by DAEMON_USER") raise SystemExit("ERROR: the grade_item.py script must be run by the DAEMON_USER") if which_machine == 'localhost': address = which_machine else: address = which_machine.split('@')[1] # prepare the zip files try: autograding_zip_tmp,submission_zip_tmp = packer_unpacker.prepare_autograding_and_submission_zip(which_machine,which_untrusted,next_directory,next_to_grade) fully_qualified_domain_name = socket.getfqdn() servername_workername = "{0}_{1}".format(fully_qualified_domain_name, address) autograding_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_autograding.zip") submission_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_submission.zip") todo_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_queue.json") with open(next_to_grade, 'r') as infile: queue_obj = json.load(infile) queue_obj["which_untrusted"] = which_untrusted queue_obj["which_machine"] = which_machine queue_obj["ship_time"] = dateutils.write_submitty_date(microseconds=True) except Exception as e: grade_items_logging.log_stack_trace(job_id=JOB_ID, trace=traceback.format_exc()) grade_items_logging.log_message(JOB_ID, message="ERROR: failed preparing submission zip or accessing next to grade "+str(e)) print("ERROR: failed preparing submission zip or accessing next to grade ", e) return False if address == "localhost": try: shutil.move(autograding_zip_tmp,autograding_zip) shutil.move(submission_zip_tmp,submission_zip) with open(todo_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) except Exception as e: grade_items_logging.log_stack_trace(job_id=JOB_ID, trace=traceback.format_exc()) grade_items_logging.log_message(JOB_ID, message="ERROR: could not move files due to the following error: "+str(e)) print("ERROR: could not move files due to the following error: {0}".format(e)) return False else: try: user, host = which_machine.split("@") ssh = paramiko.SSHClient() ssh.get_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname = host, username = user, timeout=5) sftp = ssh.open_sftp() sftp.put(autograding_zip_tmp,autograding_zip) sftp.put(submission_zip_tmp,submission_zip) with open(todo_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) sftp.put(todo_queue_file, todo_queue_file) os.remove(todo_queue_file) print("Successfully forwarded files to {0}".format(my_name)) success = True except Exception as e: grade_items_logging.log_stack_trace(job_id=JOB_ID, trace=traceback.format_exc()) grade_items_logging.log_message(JOB_ID, message="ERROR: could not move files due to the following error: "+str(e)) print("Could not move files due to the following error: {0}".format(e)) success = False finally: sftp.close() ssh.close() os.remove(autograding_zip_tmp) os.remove(submission_zip_tmp) return success # log completion of job preparation obj = packer_unpacker.load_queue_file_obj(JOB_ID,next_directory,next_to_grade) partial_path = os.path.join(obj["gradeable"],obj["who"],str(obj["version"])) item_name = os.path.join(obj["semester"],obj["course"],"submissions",partial_path) is_batch = "regrade" in obj and obj["regrade"] grade_items_logging.log_message(JOB_ID, jobname=item_name, which_untrusted=which_untrusted, is_batch=is_batch, message="Prepared job for " + which_machine) return True
def replay(starttime,endtime): replay_starttime=datetime.datetime.now() print (replay_starttime,"replay start: ",starttime) # error checking if not (starttime.year == endtime.year and starttime.month == endtime.month and starttime.day == endtime.day): print ("ERROR! invalid replay range ",starttime,"->",endtime, " (must be same day)") exit() if starttime >= endtime: print ("ERROR! invalid replay range ",starttime,"->",endtime, " (invalid times)") exit() # file the correct file file = '/var/local/submitty/logs/autograding/{:d}{:02d}{:02d}.txt'.format(starttime.year,starttime.month,starttime.day) with open(file,'r') as lines: for line in lines: things = line.split('|') original_time = dateutils.read_submitty_date(things[0]) # skip items outside of this time range if (original_time < starttime or original_time > endtime): continue # skip batch items if (things[2].strip() == "BATCH"): continue # only process the "wait" time (when we started grading the item) iswait=things[5].strip()[0:5] if (iswait != "wait:"): continue waittime=float(things[5].split()[1]) # grab the job name my_job = things[4].strip() if my_job == "": continue what = my_job.split('/') # for now, only interested in Data Structures and Computer Science 1 if not (what[1]=="csci1200" or what[1]=="csci1100"): continue # calculate when this job should be relaunched time_multipler=1.0 pause_time=replay_starttime+(time_multiplier*(original_time-starttime)) pause.until(pause_time) queue_time = dateutils.write_submitty_date() print(datetime.datetime.now()," REPLAY: ",original_time," ",my_job) # FIXME : This will need to be adjust for team assignments # and assignments with special required capabilities! item = {"semester": what[0], "course": what[1], "gradeable": what[3], "user": what[4], "team": "", "who": what[4], "is_team": False, "version": what[5], "required_capabilities": "default", "queue_time": queue_time, "regrade": True, "max_possible_grading_time" : -1 } file_name = "__".join([item['semester'], item['course'], item['gradeable'], item['who'], item['version']]) file_name = os.path.join(SUBMITTY_DATA_DIR, "to_be_graded_queue", file_name) with open(file_name, "w") as open_file: json.dump(item, open_file, sort_keys=True, indent=4) os.system("chmod o+rw {}".format(file_name)) print (datetime.datetime.now(),"replay end: ",endtime)
def main(): args = arg_parse() data_dir = os.path.join(SUBMITTY_DATA_DIR, "courses") data_dirs = data_dir.split(os.sep) grade_queue = [] if not args.times is None: starttime = dateutils.read_submitty_date(args.times[0]) endtime = dateutils.read_submitty_date(args.times[1]) replay(starttime,endtime) exit() if len(args.path) == 0: print ("ERROR! Must specify at least one path") exit() for input_path in args.path: print ('input path',input_path) # handle relative path if input_path == '.': input_path = os.getcwd() if input_path[0] != '/': input_path = os.getcwd() + '/' + input_path # remove trailing slash (if any) input_path = input_path.rstrip('/') # split the path into directories dirs = input_path.split(os.sep) # must be in the known submitty base data directory if dirs[0:len(data_dirs)] != data_dirs: print("ERROR: BAD REGRADE SUBMISSIONS PATH",input_path) raise SystemExit("You need to point to a directory within {}".format(data_dir)) # Extract directories from provided pattern path (path may be incomplete) pattern_semester="*" if len(dirs) > len(data_dirs): pattern_semester=dirs[len(data_dirs)] pattern_course="*" if len(dirs) > len(data_dirs)+1: pattern_course=dirs[len(data_dirs)+1] if len(dirs) > len(data_dirs)+2: if (dirs[len(data_dirs)+2] != "submissions"): raise SystemExit("You must specify the submissions directory within the course") pattern_gradeable="*" if len(dirs) > len(data_dirs)+3: pattern_gradeable=dirs[len(data_dirs)+3] pattern_who="*" if len(dirs) > len(data_dirs)+4: pattern_who=dirs[len(data_dirs)+4] pattern_version="*" if len(dirs) > len(data_dirs)+5: pattern_version=dirs[len(data_dirs)+5] # full pattern may include wildcards! pattern = os.path.join(pattern_semester,pattern_course,"submissions",pattern_gradeable,pattern_who,pattern_version) print("pattern: ",pattern) # Find all matching submissions for d in Path(data_dir).glob(pattern): d = str(d) if os.path.isdir(d): my_dirs = d.split(os.sep) if len(my_dirs) != len(data_dirs)+6: raise SystemExit("ERROR: directory length not as expected") # if requested, only regrade the currently active versions if args.active_only and not is_active_version(d): continue print("match: ",d) my_semester=my_dirs[len(data_dirs)] my_course=my_dirs[len(data_dirs)+1] my_gradeable=my_dirs[len(data_dirs)+3] gradeable_config = os.path.join(data_dir,my_semester,my_course,"config/build/"+"build_"+my_gradeable+".json") with open(gradeable_config, 'r') as build_configuration: datastore = json.load(build_configuration) required_capabilities = datastore.get('required_capabilities', 'default') max_grading_time = datastore.get('max_possible_grading_time', -1) #get the current time queue_time = dateutils.write_submitty_date() my_who=my_dirs[len(data_dirs)+4] my_version=my_dirs[len(data_dirs)+5] my_path=os.path.join(data_dir,my_semester,my_course,"submissions",my_gradeable,my_who,my_version) if my_path != d: raise SystemExit("ERROR: path reconstruction failed") # add them to the queue if '_' not in my_who: my_user = my_who my_team = "" my_is_team = False else: my_user = "" my_team = my_who my_is_team = True # FIXME: Set this value appropriately is_vcs_checkout = False grade_queue.append({"semester": my_semester, "course": my_course, "gradeable": my_gradeable, "user": my_user, "team": my_team, "who": my_who, "is_team": my_is_team, "version": my_version, "vcs_checkout": is_vcs_checkout, "required_capabilities" : required_capabilities, "queue_time":queue_time, "regrade":True, "max_possible_grading_time" : max_grading_time}) # Check before adding a very large number of systems to the queue if len(grade_queue) > 50 and not args.no_input: inp = input("Found {:d} matching submissions. Add to queue? [y/n]".format(len(grade_queue))) if inp.lower() not in ["yes", "y"]: raise SystemExit("Aborting...") for item in grade_queue: file_name = "__".join([item['semester'], item['course'], item['gradeable'], item['who'], item['version']]) file_name = os.path.join(SUBMITTY_DATA_DIR, "to_be_graded_queue", file_name) with open(file_name, "w") as open_file: json.dump(item, open_file, sort_keys=True, indent=4) os.system("chmod o+rw {}".format(file_name)) print("Added {:d} to the queue for regrading.".format(len(grade_queue)))
def worker_process(which_machine, address, which_untrusted, my_server): # verify the DAEMON_USER is running this script if not int(os.getuid()) == int(DAEMON_UID): autograding_utils.log_message( AUTOGRADING_LOG_PATH, JOB_ID, message="ERROR: must be run by DAEMON_USER") raise SystemExit( "ERROR: the submitty_autograding_worker.py script must be run by the DAEMON_USER" ) # ignore keyboard interrupts in the worker processes signal.signal(signal.SIGINT, signal.SIG_IGN) counter = 0 servername_workername = "{0}_{1}".format(my_server, address) autograding_zip = os.path.join( SUBMITTY_DATA_DIR, "autograding_TODO", servername_workername + "_" + which_untrusted + "_autograding.zip") submission_zip = os.path.join( SUBMITTY_DATA_DIR, "autograding_TODO", servername_workername + "_" + which_untrusted + "_submission.zip") todo_queue_file = os.path.join( SUBMITTY_DATA_DIR, "autograding_TODO", servername_workername + "_" + which_untrusted + "_queue.json") while True: if os.path.exists(todo_queue_file): try: working_directory = os.path.join( "/var/local/submitty/autograding_tmp/", which_untrusted, "tmp") results_zip_tmp = grade_item.grade_from_zip( working_directory, which_untrusted, autograding_zip, submission_zip) results_zip = os.path.join( SUBMITTY_DATA_DIR, "autograding_DONE", servername_workername + "_" + which_untrusted + "_results.zip") done_queue_file = os.path.join( SUBMITTY_DATA_DIR, "autograding_DONE", servername_workername + "_" + which_untrusted + "_queue.json") #move doesn't inherit the permissions of the destination directory. Copyfile does. shutil.copyfile(results_zip_tmp, results_zip) os.remove(results_zip_tmp) with open(todo_queue_file, 'r') as infile: queue_obj = json.load(infile) queue_obj["done_time"] = dateutils.write_submitty_date( microseconds=True) with open(done_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) except Exception as e: autograding_utils.log_message( AUTOGRADING_LOG_PATH, JOB_ID, message="ERROR attempting to unzip graded item: " + which_machine + " " + which_untrusted + ". for more details, see traces entry.") autograding_utils.log_stack_trace(AUTOGRADING_STACKTRACE_PATH, JOB_ID, trace=traceback.format_exc()) with contextlib.suppress(FileNotFoundError): os.remove(autograding_zip) with contextlib.suppress(FileNotFoundError): os.remove(submission_zip) #Respond with a failure zip file. results_zip = os.path.join( SUBMITTY_DATA_DIR, "autograding_DONE", servername_workername + "_" + which_untrusted + "_results.zip") tmp_dir = tempfile.mkdtemp() with open(os.path.join(tmp_dir, 'failure.txt'), 'w') as outfile: outfile.write("grading failed.\n") results_zip_tmp = zipfile.ZipFile(results_zip, 'w') results_zip_tmp.write(os.path.join(tmp_dir, 'failure.txt')) results_zip_tmp.close() shutil.rmtree(tmp_dir) done_queue_file = os.path.join( SUBMITTY_DATA_DIR, "autograding_DONE", servername_workername + "_" + which_untrusted + "_queue.json") with open(todo_queue_file, 'r') as infile: queue_obj = json.load(infile) queue_obj["done_time"] = dateutils.write_submitty_date( microseconds=True) with open(done_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) finally: if os.path.exists(autograding_zip): os.remove(autograding_zip) if os.path.exists(submission_zip): os.remove(submission_zip) with contextlib.suppress(FileNotFoundError): os.remove(todo_queue_file) counter = 0 else: if counter >= 10: print(which_machine, which_untrusted, "wait") counter = 0 counter += 1 time.sleep(1)
def main(): #In order to successfully re-upload the assignment, we need the directory of the assignment to be inflated, #and the semester, course name, and assignment name that it should be uploaded to. parser = argparse.ArgumentParser(description='This script was created to help professors re-upload an old semester\'s\ assignments for autograding and reevaluation. To run it, please create a new assignment via the submitty interface to\ which you wish to upload a past assignment\'s submissions. Then, run this script.') parser.add_argument('-g', '--grade', action='store_true', help='adds assignments to the grading queue.') parser.add_argument('ARCHIVED_directory', help='The path to the top level of the old assignment\'s directory tree. This folder\'s\ subdirectories should mirror a /var/local/submitty/courses/<semester>/<course>/submissions/<assignment_name> folder.') parser.add_argument('semester', help='The semester of the course you wish to upload to.') parser.add_argument('course_name', help='The name of the course you wish to upload to.') parser.add_argument('assignment_name', help='The assignment name you wish to upload to.') args = parser.parse_args() print("You are about to attempt to copy\n\tSOURCE: " + args.ARCHIVED_directory + "\n\tDESTINATION: " + SUBMITTY_DATA_DIR + "/" + args.semester+"/courses/"+args.course_name+"/submissions/"+args.assignment_name) #Check the existence of the submission path (the user should have already created the assignment for us to populate) CURRENT_course_path = os.path.join(SUBMITTY_DATA_DIR, "courses", args.semester, args.course_name) print("Current course path:" + CURRENT_course_path) CURRENT_assignment_path = os.path.join(CURRENT_course_path, "submissions", args.assignment_name) print("Current assignment path" + CURRENT_assignment_path) #we check the bin directory to see if the assignment has ever been built. (Not a guarantee, but high probability of success) CURRENT_bin_directory = os.path.join(CURRENT_course_path, "bin", args.assignment_name) if not os.path.isdir(CURRENT_bin_directory): raise SystemExit("ERROR: The directory " + CURRENT_bin_directory + " does not exist. Please make sure that you\n\t\ 1) Configured the assignment on the course website\n\t2) Did not mistype the program arguments.") else: print("SUCCESS: The directory " + CURRENT_assignment_path + " does exist.") #Make a connection to the database and grab the necessary tables. database = "submitty_" + args.semester + "_" + args.course_name print("Connecting to database: ", end="") engine = create_engine("postgresql://{}:{}@{}/{}".format(DB_USER, DB_PASS, DB_HOST, database)) conn = engine.connect() metadata = MetaData(bind=engine) print("(connection made, metadata bound)...") electronic_gradeable_data = Table("electronic_gradeable_data", metadata, autoload=True) electronic_gradeable_version = Table("electronic_gradeable_version", metadata, autoload=True) course_group = grp.getgrgid(os.stat(os.path.join(SUBMITTY_DATA_DIR,"courses",args.semester,args.course_name)).st_gid)[0] #For every user folder in our directory, for every submission folder inside of it, copy over that user/submission, add it to #the database, and create a queue file. for user_folder in sorted(os.listdir(args.ARCHIVED_directory)): print("evaluating " + user_folder) #If the item we are currently looking at is a directory, we will assume it is a student submission directory. ARCHIVED_user_dir = os.path.join(args.ARCHIVED_directory, user_folder) print("Starting work on archived dir: " + ARCHIVED_user_dir) if not os.path.isdir(ARCHIVED_user_dir): print("Skipping the following as it is not a directory: " + ARCHIVED_user_dir) continue user_name = user_folder print("processing user: "******"user_assignment_settings.json"), "r") as open_file: user_assignment_settings = json.load(open_file) #For every folder inside of the user submission folder. for submission in os.listdir(ARCHIVED_user_dir): ARCHIVED_submission_path = os.path.join(ARCHIVED_user_dir, submission) print("Evaluating submission: " + ARCHIVED_submission_path) #If this entry in the directory to be copied is not a directory, it is a user_assignments settings. #Right now, we ignore these. #TODO copy over the user_assignment_settings files and use them instead of creating new ones. if not os.path.isdir(ARCHIVED_submission_path): if user_folder == ".submit.timestamp": with open(ARCHIVED_submission_path, 'r') as old_timestamp, open(CURRENT_user_path, 'w') as new_timestamp: for line in old_timestamp: new_timestamp.write(line) os.system("chown -R submitty_php:{} {}".format(course_group, new_timestamp)) continue #if the student's submission dir does not exist, make it. if not os.path.isdir(CURRENT_user_path): os.makedirs(CURRENT_user_path) #The current directory for the new submission CURRENT_submission_path = os.path.join(CURRENT_user_path, submission) #TODO If the submission already exists, give up? if os.path.isdir(CURRENT_submission_path): print("Skipped " + CURRENT_submission_path + " as it already exists.") continue #This permission also sets the underlying submission paths recursively.g print("Set permissions on the submission") os.system("chown -R submitty_php:{} {}".format(course_group, CURRENT_user_path)) #TODO: Sort the submissions so that they are guaranteed to be given in chronological (1,2,3,etc) order. #copy in the submission directory. print("Copied from\n\tSOURCE: " + ARCHIVED_submission_path +"\n\t" + "DESTINATION: " + CURRENT_submission_path) shutil.copytree(ARCHIVED_submission_path, CURRENT_submission_path) #give the appropriate permissions os.system("chown -R submitty_php:{} {}".format(course_group, CURRENT_submission_path)) #add each submission to the database. current_time_string = dateutils.write_submitty_date() conn.execute(electronic_gradeable_data.insert(), g_id=args.assignment_name, user_id=user_name, g_version=submission, submission_time=current_time_string) #If this is the first submission, create a new entry in the table, otherwise, update. #TODO use a more reliable method of determining if this is the first submission. if int(submission) == 1: print("Entered new user " + user_name + " because submission was " + submission) conn.execute(electronic_gradeable_version.insert(), g_id=args.assignment_name, user_id=user_name, active_version=user_assignment_settings['active_version']) else: print("UPDATED: where g_id is " + args.assignment_name + " and user id is " + user_name + " to value " + str(user_assignment_settings['active_version'])) stmt = electronic_gradeable_version.update().\ where(and_(electronic_gradeable_version.c.g_id==args.assignment_name, electronic_gradeable_version.c.user_id==user_name)).\ values(active_version=user_assignment_settings['active_version']) conn.execute(stmt) with open(os.path.join(CURRENT_user_path, "user_assignment_settings.json"), "w") as open_file: json.dump(user_assignment_settings, open_file, indent = 4) if args.grade: # Create a queue file for each submission queue_file = "__".join([args.semester, args.course_name, args.assignment_name, user_name, submission]) print("Creating queue file:", queue_file) queue_file = os.path.join(SUBMITTY_DATA_DIR, "to_be_graded_queue", queue_file) with open(queue_file, "w") as open_file: # FIXME: This will need to be adjusted for team assignments # and assignments with special required capabilities! queue_time = dateutils.write_submitty_date() json.dump({"semester": args.semester, "course": args.course_name, "gradeable": args.assignment_name, "user": user_name, "team": "", "who": user_name, "is_team": False, "version": submission, "required_capabilities" : "default", "queue_time": queue_time, "regrade": True, "max_possible_grading_time": -1}, open_file) conn.close()
def prepare_job(my_name, which_machine, which_untrusted, next_directory, next_to_grade): # verify the hwcron user is running this script if not int(os.getuid()) == int(HWCRON_UID): grade_items_logging.log_message(JOB_ID, message="ERROR: must be run by hwcron") raise SystemExit( "ERROR: the grade_item.py script must be run by the hwcron user") if which_machine == 'localhost': address = which_machine else: address = which_machine.split('@')[1] # prepare the zip files try: autograding_zip_tmp, submission_zip_tmp = grade_item.prepare_autograding_and_submission_zip( which_machine, which_untrusted, next_directory, next_to_grade) fully_qualified_domain_name = socket.getfqdn() servername_workername = "{0}_{1}".format(fully_qualified_domain_name, address) autograding_zip = os.path.join( SUBMITTY_DATA_DIR, "autograding_TODO", servername_workername + "_" + which_untrusted + "_autograding.zip") submission_zip = os.path.join( SUBMITTY_DATA_DIR, "autograding_TODO", servername_workername + "_" + which_untrusted + "_submission.zip") todo_queue_file = os.path.join( SUBMITTY_DATA_DIR, "autograding_TODO", servername_workername + "_" + which_untrusted + "_queue.json") with open(next_to_grade, 'r') as infile: queue_obj = json.load(infile) queue_obj["which_untrusted"] = which_untrusted queue_obj["which_machine"] = which_machine queue_obj["ship_time"] = dateutils.write_submitty_date( microseconds=True) except Exception as e: grade_items_logging.log_message( JOB_ID, message= "ERROR: failed preparing submission zip or accessing next to grade " + str(e)) print( "ERROR: failed preparing submission zip or accessing next to grade ", e) return False if address == "localhost": try: shutil.move(autograding_zip_tmp, autograding_zip) shutil.move(submission_zip_tmp, submission_zip) with open(todo_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) except Exception as e: grade_items_logging.log_message( JOB_ID, message= "ERROR: could not move files due to the following error: " + str(e)) print( "ERROR: could not move files due to the following error: {0}". format(e)) return False else: try: user, host = which_machine.split("@") ssh = paramiko.SSHClient() ssh.get_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname=host, username=user) sftp = ssh.open_sftp() sftp.put(autograding_zip_tmp, autograding_zip) sftp.put(submission_zip_tmp, submission_zip) with open(todo_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) sftp.put(todo_queue_file, todo_queue_file) os.remove(todo_queue_file) print("Successfully forwarded files to {0}".format(my_name)) success = True except Exception as e: grade_items_logging.log_message( JOB_ID, message= "ERROR: could not move files due to the following error: " + str(e)) print( "Could not move files due to the following error: {0}".format( e)) success = False finally: sftp.close() ssh.close() os.remove(autograding_zip_tmp) os.remove(submission_zip_tmp) return success # log completion of job preparation obj = grade_item.load_queue_file_obj(JOB_ID, next_directory, next_to_grade) partial_path = os.path.join(obj["gradeable"], obj["who"], str(obj["version"])) item_name = os.path.join(obj["semester"], obj["course"], "submissions", partial_path) is_batch = "regrade" in obj and obj["regrade"] grade_items_logging.log_message(JOB_ID, jobname=item_name, which_untrusted=which_untrusted, is_batch=is_batch, message="Prepared job for " + which_machine) return True
def worker_process(which_machine, address, which_untrusted, my_server): # verify the DAEMON_USER is running this script if not int(os.getuid()) == int(DAEMON_UID): autograding_utils.log_message( AUTOGRADING_LOG_PATH, JOB_ID, message="ERROR: must be run by DAEMON_USER") raise SystemExit( "ERROR: the submitty_autograding_worker.py script must be run by the DAEMON_USER" ) # ignore keyboard interrupts in the worker processes signal.signal(signal.SIGINT, signal.SIG_IGN) counter = 0 # The full name of this worker worker_name = f"{my_server}_{address}_{which_untrusted}" # Set up key autograding_DONE directories done_dir = os.path.join(SUBMITTY_DATA_DIR, "autograding_DONE") done_queue_file = os.path.join(done_dir, f"{worker_name}_queue.json") results_zip = os.path.join(done_dir, f"{worker_name}_results.zip") # Set up key autograding_TODO directories todo_dir = os.path.join(SUBMITTY_DATA_DIR, "autograding_TODO") autograding_zip = os.path.join(todo_dir, f"{worker_name}_autograding.zip") submission_zip = os.path.join(todo_dir, f"{worker_name}_submission.zip") todo_queue_file = os.path.join(todo_dir, f"{worker_name}_queue.json") # Establish the the directory in which we will do our work working_directory = os.path.join(SUBMITTY_DATA_DIR, 'autograding_tmp', which_untrusted, "tmp") while True: if os.path.exists(todo_queue_file): try: # Attempt to grade the submission. Get back the location of the results. results_zip_tmp = grade_item.grade_from_zip( working_directory, which_untrusted, autograding_zip, submission_zip) shutil.copyfile(results_zip_tmp, results_zip) os.remove(results_zip_tmp) # At this point, we will assume that grading has progressed successfully enough to # return a coherent answer, and will say as much in the done queue file response = { 'status': 'success', 'message': 'Grading completed successfully' } except Exception: # If we threw an error while grading, log it. autograding_utils.log_message( AUTOGRADING_LOG_PATH, JOB_ID, message= f"ERROR attempting to unzip graded item: {which_machine} " f"{which_untrusted}. for more details, see traces entry.") autograding_utils.log_stack_trace(AUTOGRADING_STACKTRACE_PATH, JOB_ID, trace=traceback.format_exc()) # TODO: It is possible that autograding failed after multiple steps. # In this case, we may be able to salvage a portion of the autograding_results # directory. # Because we failed grading, we will respond with an empty results zip. results_zip_tmp = zipfile.ZipFile(results_zip, 'w') results_zip_tmp.close() # We will also respond with a done_queue_file which contains a failure message. response = { 'status': 'fail', 'message': traceback.format_exc() } finally: # Regardless of if we succeeded or failed, create a done queue file to # send to the shipper. with open(todo_queue_file, 'r') as infile: queue_obj = json.load(infile) queue_obj["done_time"] = dateutils.write_submitty_date( milliseconds=True) queue_obj['autograding_status'] = response with open(done_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) # Clean up temporary files. with contextlib.suppress(FileNotFoundError): os.remove(autograding_zip) with contextlib.suppress(FileNotFoundError): os.remove(submission_zip) with contextlib.suppress(FileNotFoundError): os.remove(todo_queue_file) counter = 0 else: if counter >= 10: print(which_machine, which_untrusted, "wait") counter = 0 counter += 1 time.sleep(1)
def prepare_job(my_name,which_machine,which_untrusted,next_directory,next_to_grade): # verify the DAEMON_USER is running this script if not int(os.getuid()) == int(DAEMON_UID): autograding_utils.log_message(AUTOGRADING_LOG_PATH, JOB_ID, message="ERROR: must be run by DAEMON_USER") raise SystemExit("ERROR: the submitty_autograding_shipper.py script must be run by the DAEMON_USER") if which_machine == 'localhost': address = which_machine else: address = which_machine.split('@')[1] # prepare the zip files try: autograding_zip_tmp,submission_zip_tmp = packer_unpacker.prepare_autograding_and_submission_zip(which_machine,which_untrusted,next_directory,next_to_grade) fully_qualified_domain_name = socket.getfqdn() servername_workername = "{0}_{1}".format(fully_qualified_domain_name, address) autograding_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_autograding.zip") submission_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_submission.zip") todo_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_queue.json") with open(next_to_grade, 'r') as infile: queue_obj = json.load(infile) queue_obj["which_untrusted"] = which_untrusted queue_obj["which_machine"] = which_machine queue_obj["ship_time"] = dateutils.write_submitty_date(microseconds=True) except Exception as e: autograding_utils.log_stack_trace(AUTOGRADING_STACKTRACE_PATH, job_id=JOB_ID, trace=traceback.format_exc()) autograding_utils.log_message(AUTOGRADING_LOG_PATH, JOB_ID, message="ERROR: failed preparing submission zip or accessing next to grade "+str(e)) print("ERROR: failed preparing submission zip or accessing next to grade ", e) return False if address == "localhost": try: shutil.move(autograding_zip_tmp,autograding_zip) shutil.move(submission_zip_tmp,submission_zip) with open(todo_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) except Exception as e: autograding_utils.log_stack_trace(AUTOGRADING_STACKTRACE_PATH, job_id=JOB_ID, trace=traceback.format_exc()) autograding_utils.log_message(AUTOGRADING_LOG_PATH, JOB_ID, message="ERROR: could not move files due to the following error: "+str(e)) print("ERROR: could not move files due to the following error: {0}".format(e)) return False else: sftp = ssh = None try: user, host = which_machine.split("@") ssh = establish_ssh_connection(my_name, user, host) sftp = ssh.open_sftp() sftp.put(autograding_zip_tmp,autograding_zip) sftp.put(submission_zip_tmp,submission_zip) with open(todo_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) sftp.put(todo_queue_file, todo_queue_file) os.remove(todo_queue_file) print("Successfully forwarded files to {0}".format(my_name)) success = True except Exception as e: autograding_utils.log_stack_trace(AUTOGRADING_STACKTRACE_PATH, job_id=JOB_ID, trace=traceback.format_exc()) autograding_utils.log_message(AUTOGRADING_LOG_PATH, JOB_ID, message="ERROR: could not move files due to the following error: "+str(e)) print("Could not move files due to the following error: {0}".format(e)) success = False finally: if sftp: sftp.close() if ssh: ssh.close() os.remove(autograding_zip_tmp) os.remove(submission_zip_tmp) return success # log completion of job preparation obj = packer_unpacker.load_queue_file_obj(JOB_ID,next_directory,next_to_grade) if "generate_output" not in obj: partial_path = os.path.join(obj["gradeable"],obj["who"],str(obj["version"])) item_name = os.path.join(obj["semester"],obj["course"],"submissions",partial_path) elif obj["generate_output"]: item_name = os.path.join(obj["semester"],obj["course"],"generated_output",obj["gradeable"]) is_batch = "regrade" in obj and obj["regrade"] autograding_utils.log_message(AUTOGRADING_LOG_PATH, JOB_ID, jobname=item_name, which_untrusted=which_untrusted, is_batch=is_batch, message="Prepared job for " + which_machine) return True
def main(): args = arg_parse() data_dir = os.path.join(SUBMITTY_DATA_DIR, "courses") data_dirs = data_dir.split(os.sep) grade_queue = [] if not args.times is None: starttime = dateutils.read_submitty_date(args.times[0]) endtime = dateutils.read_submitty_date(args.times[1]) replay(starttime,endtime) exit() if len(args.path) == 0: print ("ERROR! Must specify at least one path") exit() for input_path in args.path: print ('input path',input_path) # handle relative path if input_path == '.': input_path = os.getcwd() if input_path[0] != '/': input_path = os.getcwd() + '/' + input_path # remove trailing slash (if any) input_path = input_path.rstrip('/') # split the path into directories dirs = input_path.split(os.sep) # must be in the known submitty base data directory if dirs[0:len(data_dirs)] != data_dirs: print("ERROR: BAD REGRADE SUBMISSIONS PATH",input_path) raise SystemExit("You need to point to a directory within {}".format(data_dir)) # Extract directories from provided pattern path (path may be incomplete) pattern_semester="*" if len(dirs) > len(data_dirs): pattern_semester=dirs[len(data_dirs)] pattern_course="*" if len(dirs) > len(data_dirs)+1: pattern_course=dirs[len(data_dirs)+1] if len(dirs) > len(data_dirs)+2: if (dirs[len(data_dirs)+2] != "submissions"): raise SystemExit("You must specify the submissions directory within the course") pattern_gradeable="*" if len(dirs) > len(data_dirs)+3: pattern_gradeable=dirs[len(data_dirs)+3] pattern_who="*" if len(dirs) > len(data_dirs)+4: pattern_who=dirs[len(data_dirs)+4] pattern_version="*" if len(dirs) > len(data_dirs)+5: pattern_version=dirs[len(data_dirs)+5] # full pattern may include wildcards! pattern = os.path.join(data_dir,pattern_semester,pattern_course,"submissions",pattern_gradeable,pattern_who,pattern_version) print("pattern: ",pattern) # Find all matching submissions for d in glob.glob(pattern): if os.path.isdir(d): my_dirs = d.split(os.sep) if len(my_dirs) != len(data_dirs)+6: raise SystemExit("ERROR: directory length not as expected") # if requested, only regrade the currently active versions if args.active_only and not is_active_version(d): continue print("match: ",d) my_semester=my_dirs[len(data_dirs)] my_course=my_dirs[len(data_dirs)+1] my_gradeable=my_dirs[len(data_dirs)+3] gradeable_config = os.path.join(data_dir,my_semester,my_course,"config/build/"+"build_"+my_gradeable+".json") with open(gradeable_config, 'r') as build_configuration: datastore = json.load(build_configuration) required_capabilities = datastore.get('required_capabilities', 'default') max_grading_time = datastore.get('max_possible_grading_time', -1) #get the current time queue_time = dateutils.write_submitty_date() my_who=my_dirs[len(data_dirs)+4] my_version=my_dirs[len(data_dirs)+5] my_path=os.path.join(data_dir,my_semester,my_course,"submissions",my_gradeable,my_who,my_version) if my_path != d: raise SystemExit("ERROR: path reconstruction failed") # add them to the queue if '_' not in my_who: my_user = my_who my_team = "" my_is_team = False else: my_user = "" my_team = my_who my_is_team = True grade_queue.append({"semester": my_semester, "course": my_course, "gradeable": my_gradeable, "user": my_user, "team": my_team, "who": my_who, "is_team": my_is_team, "version": my_version, "required_capabilities" : required_capabilities, "queue_time":queue_time, "regrade":True, "max_possible_grading_time" : max_grading_time}) # Check before adding a very large number of systems to the queue if len(grade_queue) > 50 and not args.no_input: inp = input("Found {:d} matching submissions. Add to queue? [y/n]".format(len(grade_queue))) if inp.lower() not in ["yes", "y"]: raise SystemExit("Aborting...") for item in grade_queue: file_name = "__".join([item['semester'], item['course'], item['gradeable'], item['who'], item['version']]) file_name = os.path.join(SUBMITTY_DATA_DIR, "to_be_graded_queue", file_name) with open(file_name, "w") as open_file: json.dump(item, open_file, sort_keys=True, indent=4) os.system("chmod o+rw {}".format(file_name)) print("Added {:d} to the queue for regrading.".format(len(grade_queue)))
def prepare_autograding_and_submission_zip(which_machine,which_untrusted,next_directory,next_to_grade): os.chdir(SUBMITTY_DATA_DIR) # generate a random id to be used to track this job in the autograding logs job_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(6)) # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = load_queue_file_obj(job_id,next_directory,next_to_grade) partial_path = os.path.join(obj["gradeable"],obj["who"],str(obj["version"])) item_name = os.path.join(obj["semester"],obj["course"],"submissions",partial_path) submission_path = os.path.join(SUBMITTY_DATA_DIR,"courses",item_name) if not os.path.isdir(submission_path): grade_items_logging.log_message(job_id, message="ERROR: the submission directory does not exist " + submission_path) raise RuntimeError("ERROR: the submission directory does not exist", submission_path) print(which_machine,which_untrusted,"prepare zip",submission_path) is_vcs,vcs_type,vcs_base_url,vcs_subdirectory = get_vcs_info(SUBMITTY_DATA_DIR,obj["semester"],obj["course"],obj["gradeable"],obj["who"],obj["team"]) is_batch_job = "regrade" in obj and obj["regrade"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory,next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = (grading_began-queue_time).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,"zip",item_name,"wait:",waittime,"") # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"provided_code",obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_input",obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_output",obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"custom_validation_code",obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"bin",obj["gradeable"]) form_json_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","form","form_"+obj["gradeable"]+".json") complete_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","complete_config","complete_config_"+obj["gradeable"]+".json") if not os.path.exists(form_json_config): grade_items_logging.log_message(job_id,message="ERROR: the form json file does not exist " + form_json_config) raise RuntimeError("ERROR: the form json file does not exist ",form_json_config) if not os.path.exists(complete_config): grade_items_logging.log_message(job_id,message="ERROR: the complete config file does not exist " + complete_config) raise RuntimeError("ERROR: the complete config file does not exist ",complete_config) # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = tempfile.mkdtemp() tmp_autograding = os.path.join(tmp,"TMP_AUTOGRADING") os.mkdir(tmp_autograding) tmp_submission = os.path.join(tmp,"TMP_SUBMISSION") os.mkdir(tmp_submission) copytree_if_exists(provided_code_path,os.path.join(tmp_autograding,"provided_code")) copytree_if_exists(test_input_path,os.path.join(tmp_autograding,"test_input")) copytree_if_exists(test_output_path,os.path.join(tmp_autograding,"test_output")) copytree_if_exists(custom_validation_code_path,os.path.join(tmp_autograding,"custom_validation_code")) copytree_if_exists(bin_path,os.path.join(tmp_autograding,"bin")) shutil.copy(form_json_config,os.path.join(tmp_autograding,"form.json")) shutil.copy(complete_config,os.path.join(tmp_autograding,"complete_config.json")) checkout_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"checkout",partial_path) results_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"results",partial_path) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path,"history.json") history_file_tmp = "" if os.path.isfile(history_file): shutil.copy(history_file,os.path.join(tmp_submission,"history.json")) # get info from the gradeable config file with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get("use_checkout_subdirectory","") checkout_subdir_path = os.path.join(checkout_path,checkout_subdirectory) queue_file = os.path.join(next_directory,next_to_grade) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp,"TMP_SUBMISSION","tmp_logs") os.makedirs(tmp_logs) # 'touch' a file in the logs folder open(os.path.join(tmp_logs,"overall.txt"), 'a') # -------------------------------------------------------------------- # CONFIRM WE HAVE A CHECKOUT OF THE STUDENT'S REPO if is_vcs: # there should be a checkout log file in the results directory # move that file to the tmp logs directory.. vcs_checkout_logfile = os.path.join(results_path,"logs","vcs_checkout.txt") if os.path.isfile(vcs_checkout_logfile): shutil.move(vcs_checkout_logfile,tmp_logs) else: grade_items_logging.log_message(JOB_ID, message=str(my_name)+" ERROR: missing vcs_checkout.txt logfile "+str(vcs_checkout_logfile)) copytree_if_exists(submission_path,os.path.join(tmp_submission,"submission")) copytree_if_exists(checkout_path,os.path.join(tmp_submission,"checkout")) obj["queue_time"] = queue_time_longstring obj["regrade"] = is_batch_job obj["waittime"] = waittime obj["job_id"] = job_id with open(os.path.join(tmp_submission,"queue_file.json"),'w') as outfile: json.dump(obj,outfile,sort_keys=True,indent=4,separators=(',', ': ')) grading_began_longstring = dateutils.write_submitty_date(grading_began) with open(os.path.join(tmp_submission,".grading_began"), 'w') as f: print (grading_began_longstring,file=f) # zip up autograding & submission folders filehandle1, my_autograding_zip_file =tempfile.mkstemp() filehandle2, my_submission_zip_file =tempfile.mkstemp() grade_item.zip_my_directory(tmp_autograding,my_autograding_zip_file) grade_item.zip_my_directory(tmp_submission,my_submission_zip_file) os.close(filehandle1) os.close(filehandle2) # cleanup shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp) #grade_items_logging.log_message(job_id,is_batch_job,"done zip",item_name) return (my_autograding_zip_file,my_submission_zip_file)
def prepare_autograding_and_submission_zip(which_machine, which_untrusted, next_directory, next_to_grade): os.chdir(SUBMITTY_DATA_DIR) # generate a random id to be used to track this job in the autograding logs job_id = ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(6)) # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = load_queue_file_obj(job_id, next_directory, next_to_grade) partial_path = os.path.join(obj["gradeable"], obj["who"], str(obj["version"])) item_name = os.path.join(obj["semester"], obj["course"], "submissions", partial_path) submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name) if not os.path.isdir(submission_path): grade_items_logging.log_message( job_id, message="ERROR: the submission directory does not exist " + submission_path) raise RuntimeError("ERROR: the submission directory does not exist", submission_path) print(which_machine, which_untrusted, "prepare zip", submission_path) is_vcs, vcs_type, vcs_base_url, vcs_subdirectory = get_vcs_info( SUBMITTY_DATA_DIR, obj["semester"], obj["course"], obj["gradeable"], obj["who"], obj["team"]) is_batch_job = "regrade" in obj and obj["regrade"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory, next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = (grading_began - queue_time).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, "zip", item_name, "wait:", waittime, "") # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "provided_code", obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_input", obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_output", obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "custom_validation_code", obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "bin", obj["gradeable"]) form_json_config = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "form", "form_" + obj["gradeable"] + ".json") complete_config = os.path.join( SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "complete_config", "complete_config_" + obj["gradeable"] + ".json") if not os.path.exists(form_json_config): grade_items_logging.log_message( job_id, message="ERROR: the form json file does not exist " + form_json_config) raise RuntimeError("ERROR: the form json file does not exist ", form_json_config) if not os.path.exists(complete_config): grade_items_logging.log_message( job_id, message="ERROR: the complete config file does not exist " + complete_config) raise RuntimeError("ERROR: the complete config file does not exist ", complete_config) # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = tempfile.mkdtemp() tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") os.mkdir(tmp_autograding) tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") os.mkdir(tmp_submission) copytree_if_exists(provided_code_path, os.path.join(tmp_autograding, "provided_code")) copytree_if_exists(test_input_path, os.path.join(tmp_autograding, "test_input")) copytree_if_exists(test_output_path, os.path.join(tmp_autograding, "test_output")) copytree_if_exists(custom_validation_code_path, os.path.join(tmp_autograding, "custom_validation_code")) copytree_if_exists(bin_path, os.path.join(tmp_autograding, "bin")) shutil.copy(form_json_config, os.path.join(tmp_autograding, "form.json")) shutil.copy(complete_config, os.path.join(tmp_autograding, "complete_config.json")) checkout_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "checkout", partial_path) results_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "results", partial_path) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path, "history.json") history_file_tmp = "" if os.path.isfile(history_file): shutil.copy(history_file, os.path.join(tmp_submission, "history.json")) # get info from the gradeable config file with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) queue_file = os.path.join(next_directory, next_to_grade) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") os.makedirs(tmp_logs) # 'touch' a file in the logs folder open(os.path.join(tmp_logs, "overall.txt"), 'a') # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) # -------------------------------------------------------------------- # CHECKOUT THE STUDENT's REPO if is_vcs: # cleanup the previous checkout (if it exists) shutil.rmtree(checkout_path, ignore_errors=True) os.makedirs(checkout_path, exist_ok=True) try: # If we are public or private github, we will have an empty vcs_subdirectory if vcs_subdirectory == '': with open(os.path.join( submission_path, ".submit.VCS_CHECKOUT")) as submission_vcs_file: VCS_JSON = json.load(submission_vcs_file) git_user_id = VCS_JSON["git_user_id"] git_repo_id = VCS_JSON["git_repo_id"] if not valid_github_user_id(git_user_id): raise Exception( "Invalid GitHub user/organization name: '" + git_user_id + "'") if not valid_github_repo_id(git_repo_id): raise Exception("Invalid GitHub repository name: '" + git_repo_id + "'") # construct path for GitHub vcs_path = "https://www.github.com/" + git_user_id + "/" + git_repo_id # is vcs_subdirectory standalone or should it be combined with base_url? elif vcs_subdirectory[0] == '/' or '://' in vcs_subdirectory: vcs_path = vcs_subdirectory else: if '://' in vcs_base_url: vcs_path = urllib.parse.urljoin(vcs_base_url, vcs_subdirectory) else: vcs_path = os.path.join(vcs_base_url, vcs_subdirectory) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nVCS CHECKOUT", file=f) print('vcs_base_url', vcs_base_url, file=f) print('vcs_subdirectory', vcs_subdirectory, file=f) print('vcs_path', vcs_path, file=f) print(['/usr/bin/git', 'clone', vcs_path, checkout_path], file=f) # git clone may fail -- because repository does not exist, # or because we don't have appropriate access credentials try: subprocess.check_call( ['/usr/bin/git', 'clone', vcs_path, checkout_path]) os.chdir(checkout_path) # determine which version we need to checkout # if the repo is empty or the master branch does not exist, this command will fail try: what_version = subprocess.check_output([ 'git', 'rev-list', '-n', '1', '--before="' + submission_string + '"', 'master' ]) what_version = str(what_version.decode('utf-8')).rstrip() if what_version == "": # oops, pressed the grade button before a valid commit shutil.rmtree(checkout_path, ignore_errors=True) else: # and check out the right version subprocess.call( ['git', 'checkout', '-b', 'grade', what_version]) os.chdir(tmp) subprocess.call(['ls', '-lR', checkout_path], stdout=open(tmp_logs + "/overall.txt", 'a')) obj['revision'] = what_version # exception on git rev-list except subprocess.CalledProcessError as error: grade_items_logging.log_message( job_id, message= "ERROR: failed to determine version on master branch " + str(error)) os.chdir(checkout_path) with open( os.path.join( checkout_path, "failed_to_determine_version_on_master_branch.txt" ), 'w') as f: print(str(error), file=f) print("\n", file=f) print( "Check to be sure the repository is not empty.\n", file=f) print( "Check to be sure the repository has a master branch.\n", file=f) print( "And check to be sure the timestamps on the master branch are reasonable.\n", file=f) # exception on git clone except subprocess.CalledProcessError as error: grade_items_logging.log_message( job_id, message="ERROR: failed to clone repository " + str(error)) os.chdir(checkout_path) with open( os.path.join(checkout_path, "failed_to_clone_repository.txt"), 'w') as f: print(str(error), file=f) print("\n", file=f) print("Check to be sure the repository exists.\n", file=f) print( "And check to be sure the submitty_daemon user has appropriate access credentials.\n", file=f) # exception in constructing full git repository url/path except Exception as error: grade_items_logging.log_message( job_id, message="ERROR: failed to construct valid repository url/path" + str(error)) os.chdir(checkout_path) with open( os.path.join( checkout_path, "failed_to_construct_valid_repository_url.txt"), 'w') as f: print(str(error), file=f) print("\n", file=f) print("Check to be sure the repository exists.\n", file=f) print( "And check to be sure the submitty_daemon user has appropriate access credentials.\n", file=f) copytree_if_exists(submission_path, os.path.join(tmp_submission, "submission")) copytree_if_exists(checkout_path, os.path.join(tmp_submission, "checkout")) obj["queue_time"] = queue_time_longstring obj["regrade"] = is_batch_job obj["waittime"] = waittime obj["job_id"] = job_id with open(os.path.join(tmp_submission, "queue_file.json"), 'w') as outfile: json.dump(obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) grading_began_longstring = dateutils.write_submitty_date(grading_began) with open(os.path.join(tmp_submission, ".grading_began"), 'w') as f: print(grading_began_longstring, file=f) # zip up autograding & submission folders filehandle1, my_autograding_zip_file = tempfile.mkstemp() filehandle2, my_submission_zip_file = tempfile.mkstemp() grade_item.zip_my_directory(tmp_autograding, my_autograding_zip_file) grade_item.zip_my_directory(tmp_submission, my_submission_zip_file) os.close(filehandle1) os.close(filehandle2) # cleanup shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp) #grade_items_logging.log_message(job_id,is_batch_job,"done zip",item_name) return (my_autograding_zip_file, my_submission_zip_file)
def grade_from_zip(my_autograding_zip_file, my_submission_zip_file, which_untrusted): os.chdir(SUBMITTY_DATA_DIR) tmp = os.path.join("/var/local/submitty/autograding_tmp/", which_untrusted, "tmp") # clean up old usage of this directory shutil.rmtree(tmp, ignore_errors=True) os.makedirs(tmp) which_machine = socket.gethostname() # unzip autograding and submission folders tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") unzip_this_file(my_autograding_zip_file, tmp_autograding) unzip_this_file(my_submission_zip_file, tmp_submission) os.remove(my_autograding_zip_file) os.remove(my_submission_zip_file) tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") queue_file = os.path.join(tmp_submission, "queue_file.json") with open(queue_file, 'r') as infile: queue_obj = json.load(infile) queue_time_longstring = queue_obj["queue_time"] waittime = queue_obj["waittime"] is_batch_job = queue_obj["regrade"] job_id = queue_obj["job_id"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" partial_path = os.path.join(queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"], queue_obj["course"], "submissions", partial_path) grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "wait:", waittime, "") # -------------------------------------------------------------------- # START DOCKER # WIP: This option file facilitated testing... #USE_DOCKER = os.path.isfile("/tmp/use_docker") #use_docker_string="grading begins, using DOCKER" if USE_DOCKER else "grading begins (not using docker)" #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message=use_docker_string) container = None if USE_DOCKER: container = subprocess.check_output([ 'docker', 'run', '-t', '-d', '-v', tmp + ':' + tmp, 'ubuntu:custom' ]).decode('utf8').strip() dockerlaunch_done = dateutils.get_current_time() dockerlaunch_time = (dockerlaunch_done - grading_began).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, submission_path, "dcct:", dockerlaunch_time, "docker container created") # -------------------------------------------------------------------- # COMPILE THE SUBMITTED CODE with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nCOMPILATION STARTS", file=f) # copy submitted files to the tmp compilation directory tmp_compilation = os.path.join(tmp, "TMP_COMPILATION") os.mkdir(tmp_compilation) os.chdir(tmp_compilation) submission_path = os.path.join(tmp_submission, "submission") checkout_path = os.path.join(tmp_submission, "checkout") provided_code_path = os.path.join(tmp_autograding, "provided_code") test_input_path = os.path.join(tmp_autograding, "test_input") test_output_path = os.path.join(tmp_autograding, "test_output") custom_validation_code_path = os.path.join(tmp_autograding, "custom_validation_code") bin_path = os.path.join(tmp_autograding, "bin") form_json_config = os.path.join(tmp_autograding, "form.json") complete_config = os.path.join(tmp_autograding, "complete_config.json") with open(form_json_config, 'r') as infile: gradeable_config_obj = json.load(infile) gradeable_deadline_string = gradeable_config_obj["date_due"] with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) patterns_submission_to_compilation = complete_config_obj["autograding"][ "submission_to_compilation"] pattern_copy("submission_to_compilation", patterns_submission_to_compilation, submission_path, tmp_compilation, tmp_logs) is_vcs = gradeable_config_obj["upload_type"] == "repository" checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) if is_vcs: pattern_copy("checkout_to_compilation", patterns_submission_to_compilation, checkout_subdir_path, tmp_compilation, tmp_logs) # copy any instructor provided code files to tmp compilation directory copy_contents_into(job_id, provided_code_path, tmp_compilation, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy compile.out to the current directory shutil.copy(os.path.join(bin_path, "compile.out"), os.path.join(tmp_compilation, "my_compile.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_compilation, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP) add_permissions(tmp, stat.S_IROTH | stat.S_IXOTH) add_permissions(tmp_logs, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() with open(os.path.join(tmp_logs, "compilation_log.txt"), 'w') as logfile: if USE_DOCKER: compile_success = subprocess.call([ 'docker', 'exec', '-w', tmp_compilation, container, os.path.join(tmp_compilation, 'my_compile.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string ], stdout=logfile) else: compile_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(tmp_compilation, "my_compile.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string ], stdout=logfile) if compile_success == 0: print(which_machine, which_untrusted, "COMPILATION OK") else: print(which_machine, which_untrusted, "COMPILATION FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="COMPILATION FAILURE") untrusted_grant_rwx_access(which_untrusted, tmp_compilation) # remove the compilation program os.remove(os.path.join(tmp_compilation, "my_compile.out")) # return to the main tmp directory os.chdir(tmp) # -------------------------------------------------------------------- # make the runner directory with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nRUNNER STARTS", file=f) tmp_work = os.path.join(tmp, "TMP_WORK") os.makedirs(tmp_work) os.chdir(tmp_work) # move all executable files from the compilation directory to the main tmp directory # Note: Must preserve the directory structure of compiled files (esp for Java) patterns_submission_to_runner = complete_config_obj["autograding"][ "submission_to_runner"] pattern_copy("submission_to_runner", patterns_submission_to_runner, submission_path, tmp_work, tmp_logs) if is_vcs: pattern_copy("checkout_to_runner", patterns_submission_to_runner, checkout_subdir_path, tmp_work, tmp_logs) patterns_compilation_to_runner = complete_config_obj["autograding"][ "compilation_to_runner"] pattern_copy("compilation_to_runner", patterns_compilation_to_runner, tmp_compilation, tmp_work, tmp_logs) # copy input files to tmp_work directory copy_contents_into(job_id, test_input_path, tmp_work, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy runner.out to the current directory shutil.copy(os.path.join(bin_path, "run.out"), os.path.join(tmp_work, "my_runner.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # run the run.out as the untrusted user with open(os.path.join(tmp_logs, "runner_log.txt"), 'w') as logfile: print("LOGGING BEGIN my_runner.out", file=logfile) logfile.flush() try: if USE_DOCKER: runner_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_runner.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string ], stdout=logfile) else: runner_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(tmp_work, "my_runner.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string ], stdout=logfile) logfile.flush() except Exception as e: print("ERROR caught runner.out exception={0}".format(str( e.args[0])).encode("utf-8"), file=logfile) logfile.flush() print("LOGGING END my_runner.out", file=logfile) logfile.flush() killall_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "killall.py") ], stdout=logfile) print("KILLALL COMPLETE my_runner.out", file=logfile) logfile.flush() if killall_success != 0: msg = 'RUNNER ERROR: had to kill {} process(es)'.format( killall_success) print("pid", os.getpid(), msg) grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "", "", msg) if runner_success == 0: print(which_machine, which_untrusted, "RUNNER OK") else: print(which_machine, which_untrusted, "RUNNER FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="RUNNER FAILURE") untrusted_grant_rwx_access(which_untrusted, tmp_work) untrusted_grant_rwx_access(which_untrusted, tmp_compilation) # -------------------------------------------------------------------- # RUN VALIDATOR with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nVALIDATION STARTS", file=f) # copy results files from compilation... patterns_submission_to_validation = complete_config_obj["autograding"][ "submission_to_validation"] pattern_copy("submission_to_validation", patterns_submission_to_validation, submission_path, tmp_work, tmp_logs) if is_vcs: pattern_copy("checkout_to_validation", patterns_submission_to_validation, checkout_subdir_path, tmp_work, tmp_logs) patterns_compilation_to_validation = complete_config_obj["autograding"][ "compilation_to_validation"] pattern_copy("compilation_to_validation", patterns_compilation_to_validation, tmp_compilation, tmp_work, tmp_logs) # remove the compilation directory shutil.rmtree(tmp_compilation) # copy output files to tmp_work directory copy_contents_into(job_id, test_output_path, tmp_work, tmp_logs) # copy any instructor custom validation code into the tmp work directory copy_contents_into(job_id, custom_validation_code_path, tmp_work, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy validator.out to the current directory shutil.copy(os.path.join(bin_path, "validate.out"), os.path.join(tmp_work, "my_validator.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(os.path.join(tmp_work, "my_validator.out"), stat.S_IROTH | stat.S_IXOTH) # validator the validator.out as the untrusted user with open(os.path.join(tmp_logs, "validator_log.txt"), 'w') as logfile: if USE_DOCKER: validator_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_validator.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string ], stdout=logfile) else: validator_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(tmp_work, "my_validator.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string ], stdout=logfile) if validator_success == 0: print(which_machine, which_untrusted, "VALIDATOR OK") else: print(which_machine, which_untrusted, "VALIDATOR FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="VALIDATION FAILURE") untrusted_grant_rwx_access(which_untrusted, tmp_work) # grab the result of autograding grade_result = "" with open(os.path.join(tmp_work, "grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line # -------------------------------------------------------------------- # MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE tmp_results = os.path.join(tmp, "TMP_RESULTS") with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nARCHIVING STARTS", file=f) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) os.makedirs(os.path.join(tmp_results, "details")) patterns_work_to_details = complete_config_obj["autograding"][ "work_to_details"] pattern_copy("work_to_details", patterns_work_to_details, tmp_work, os.path.join(tmp_results, "details"), tmp_logs) history_file_tmp = os.path.join(tmp_submission, "history.json") history_file = os.path.join(tmp_results, "history.json") if os.path.isfile(history_file_tmp): shutil.move(history_file_tmp, history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file, int(HWCRON_UID), ta_group_id) add_permissions(history_file, stat.S_IRGRP) grading_finished = dateutils.get_current_time() shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results) # ------------------------------------------------------------- # create/append to the results history # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) gradeable_deadline_datetime = dateutils.read_submitty_date( gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date( gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int( (submission_datetime - gradeable_deadline_datetime).total_seconds()) # note: negative = not late with open(os.path.join(tmp_submission, ".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) grading_finished_longstring = dateutils.write_submitty_date( grading_finished) gradingtime = (grading_finished - grading_began).total_seconds() with open(os.path.join(tmp_submission, "queue_file.json"), 'r') as infile: queue_obj = json.load(infile) queue_obj["gradingtime"] = gradingtime queue_obj["grade_result"] = grade_result queue_obj["which_untrusted"] = which_untrusted with open(os.path.join(tmp_results, "queue_file.json"), 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) with open(os.path.join(tmp_work, "results.json"), 'r') as read_file: results_obj = json.load(read_file) if 'revision' in queue_obj.keys(): results_obj['revision'] = queue_obj['revision'] with open(os.path.join(tmp_results, "results.json"), 'w') as outfile: json.dump(results_obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) write_grade_history.just_write_grade_history( history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_time_longstring, is_batch_job_string, grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result) os.chdir(SUBMITTY_DATA_DIR) if USE_DOCKER: with open(os.path.join(tmp_logs, "overall_log.txt"), 'w') as logfile: chmod_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, 'chmod', '-R', 'o+rwx', '.' ], stdout=logfile) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: f.write("FINISHED GRADING!\n") # save the logs! shutil.copytree(tmp_logs, os.path.join(tmp_results, "logs")) # zip up results folder filehandle, my_results_zip_file = tempfile.mkstemp() zip_my_directory(tmp_results, my_results_zip_file) os.close(filehandle) shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp_results) shutil.rmtree(tmp_work) shutil.rmtree(tmp) # WIP: extra logging for testing #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message="done grading") # -------------------------------------------------------------------- # CLEAN UP DOCKER if USE_DOCKER: subprocess.call(['docker', 'rm', '-f', container]) dockerdestroy_done = dateutils.get_current_time() dockerdestroy_time = (dockerdestroy_done - grading_finished).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, submission_path, "ddt:", dockerdestroy_time, "docker container destroyed") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "grade:", gradingtime, grade_result) return my_results_zip_file
def grade_from_zip(my_autograding_zip_file,my_submission_zip_file,which_untrusted): os.chdir(SUBMITTY_DATA_DIR) tmp = os.path.join("/var/local/submitty/autograding_tmp/",which_untrusted,"tmp") if os.path.exists(tmp): untrusted_grant_rwx_access(which_untrusted, tmp) add_permissions_recursive(tmp, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # Remove any and all containers left over from past runs. old_containers = subprocess.check_output(['docker', 'ps', '-aq', '-f', 'name={0}'.format(which_untrusted)]).split() for old_container in old_containers: subprocess.call(['docker', 'rm', '-f', old_container.decode('utf8')]) # clean up old usage of this directory shutil.rmtree(tmp,ignore_errors=True) os.mkdir(tmp) which_machine=socket.gethostname() # unzip autograding and submission folders tmp_autograding = os.path.join(tmp,"TMP_AUTOGRADING") tmp_submission = os.path.join(tmp,"TMP_SUBMISSION") try: unzip_this_file(my_autograding_zip_file,tmp_autograding) unzip_this_file(my_submission_zip_file,tmp_submission) except: raise os.remove(my_autograding_zip_file) os.remove(my_submission_zip_file) tmp_logs = os.path.join(tmp,"TMP_SUBMISSION","tmp_logs") queue_file = os.path.join(tmp_submission,"queue_file.json") with open(queue_file, 'r') as infile: queue_obj = json.load(infile) queue_time_longstring = queue_obj["queue_time"] waittime = queue_obj["waittime"] is_batch_job = queue_obj["regrade"] job_id = queue_obj["job_id"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" revision = queue_obj.get("revision", None) partial_path = os.path.join(queue_obj["gradeable"],queue_obj["who"],str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"],queue_obj["course"],"submissions",partial_path) grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,"wait:",waittime,"") with open(os.path.join(tmp_submission,".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) submission_path = os.path.join(tmp_submission, "submission") checkout_path = os.path.join(tmp_submission, "checkout") provided_code_path = os.path.join(tmp_autograding, "provided_code") test_input_path = os.path.join(tmp_autograding, "test_input") test_output_path = os.path.join(tmp_autograding, "test_output") custom_validation_code_path = os.path.join(tmp_autograding, "custom_validation_code") bin_path = os.path.join(tmp_autograding, "bin") form_json_config = os.path.join(tmp_autograding, "form.json") complete_config = os.path.join(tmp_autograding, "complete_config.json") with open(form_json_config, 'r') as infile: gradeable_config_obj = json.load(infile) gradeable_deadline_string = gradeable_config_obj["date_due"] with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) is_vcs = gradeable_config_obj["upload_type"] == "repository" checkout_subdirectory = complete_config_obj["autograding"].get("use_checkout_subdirectory","") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) if complete_config_obj.get('one_part_only', False): allow_only_one_part(submission_path, os.path.join(tmp_logs, "overall.txt")) if is_vcs: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("WARNING: ONE_PART_ONLY OPTION DOES NOT MAKE SENSE WITH VCS SUBMISSION", file=f) # -------------------------------------------------------------------- # START DOCKER # NOTE: DOCKER SUPPORT PRELIMINARY -- NEEDS MORE SECURITY BEFORE DEPLOYED ON LIVE SERVER complete_config = os.path.join(tmp_autograding,"complete_config.json") with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) # Save ourselves if autograding_method is None. autograding_method = complete_config_obj.get("autograding_method", "") USE_DOCKER = True if autograding_method == "docker" else False # -------------------------------------------------------------------- # COMPILE THE SUBMITTED CODE with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nCOMPILATION STARTS", file=f) # copy submitted files to the tmp compilation directory tmp_compilation = os.path.join(tmp,"TMP_COMPILATION") os.mkdir(tmp_compilation) os.chdir(tmp_compilation) submission_path = os.path.join(tmp_submission,"submission") checkout_path = os.path.join(tmp_submission,"checkout") provided_code_path = os.path.join(tmp_autograding,"provided_code") test_input_path = os.path.join(tmp_autograding,"test_input") test_output_path = os.path.join(tmp_autograding,"test_output") custom_validation_code_path = os.path.join(tmp_autograding,"custom_validation_code") bin_path = os.path.join(tmp_autograding,"bin") form_json_config = os.path.join(tmp_autograding,"form.json") with open(form_json_config, 'r') as infile: gradeable_config_obj = json.load(infile) gradeable_deadline_string = gradeable_config_obj["date_due"] patterns_submission_to_compilation = complete_config_obj["autograding"]["submission_to_compilation"] add_permissions(tmp_logs,stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) if USE_DOCKER: print("!!!!!!!!!!!!!!!!!!USING DOCKER!!!!!!!!!!!!!!!!!!!!!!!!") with open(complete_config, 'r') as infile: config = json.load(infile) my_testcases = config['testcases'] # grab the submission time with open(os.path.join(submission_path,".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() with open(os.path.join(tmp_logs,"compilation_log.txt"), 'w') as logfile: # we start counting from one. executable_path_list = list() for testcase_num in range(1, len(my_testcases)+1): testcase_folder = os.path.join(tmp_compilation, "test{:02}".format(testcase_num)) if 'type' in my_testcases[testcase_num-1]: if my_testcases[testcase_num-1]['type'] != 'FileCheck' and my_testcases[testcase_num-1]['type'] != 'Compilation': continue if my_testcases[testcase_num-1]['type'] == 'Compilation': if 'executable_name' in my_testcases[testcase_num-1]: provided_executable_list = my_testcases[testcase_num-1]['executable_name'] if not isinstance(provided_executable_list, (list,)): provided_executable_list = list([provided_executable_list]) for executable_name in provided_executable_list: if executable_name.strip() == '': continue executable_path = os.path.join(testcase_folder, executable_name) executable_path_list.append((executable_path, executable_name)) else: continue os.makedirs(testcase_folder) pattern_copy("submission_to_compilation",patterns_submission_to_compilation,submission_path,testcase_folder,tmp_logs) if is_vcs: pattern_copy("checkout_to_compilation",patterns_submission_to_compilation,checkout_subdir_path,testcase_folder,tmp_logs) # copy any instructor provided code files to tmp compilation directory copy_contents_into(job_id,provided_code_path,testcase_folder,tmp_logs) # copy compile.out to the current directory shutil.copy (os.path.join(bin_path,"compile.out"),os.path.join(testcase_folder,"my_compile.out")) add_permissions(os.path.join(testcase_folder,"my_compile.out"), stat.S_IXUSR | stat.S_IXGRP |stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #untrusted_grant_rwx_access(which_untrusted, tmp_compilation) untrusted_grant_rwx_access(which_untrusted, testcase_folder) add_permissions_recursive(testcase_folder, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) if USE_DOCKER: try: #There can be only one container for a compilation step, so grab its container image #TODO: set default in load_config_json.cpp if my_testcases[testcase_num-1]['type'] == 'FileCheck': print("performing filecheck in default ubuntu:custom container") container_image = "ubuntu:custom" else: container_image = my_testcases[testcase_num-1]["containers"][0]["container_image"] print('creating a compilation container with image {0}'.format(container_image)) untrusted_uid = str(getpwnam(which_untrusted).pw_uid) compilation_container = None compilation_container = subprocess.check_output(['docker', 'create', '-i', '-u', untrusted_uid, '--network', 'none', '-v', testcase_folder + ':' + testcase_folder, '-w', testcase_folder, container_image, #The command to be run. os.path.join(testcase_folder, 'my_compile.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string, '--testcase', str(testcase_num) ]).decode('utf8').strip() print("starting container") compile_success = subprocess.call(['docker', 'start', '-i', compilation_container], stdout=logfile, cwd=testcase_folder) except Exception as e: print('An error occurred when compiling with docker.') grade_items_logging.log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) finally: if compilation_container != None: subprocess.call(['docker', 'rm', '-f', compilation_container]) print("cleaned up compilation container.") else: compile_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(testcase_folder,"my_compile.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string, '--testcase', str(testcase_num)], stdout=logfile, cwd=testcase_folder) # remove the compilation program untrusted_grant_rwx_access(which_untrusted, testcase_folder) os.remove(os.path.join(testcase_folder,"my_compile.out")) if compile_success == 0: print (which_machine,which_untrusted,"COMPILATION OK") else: print (which_machine,which_untrusted,"COMPILATION FAILURE") grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="COMPILATION FAILURE") add_permissions_recursive(tmp_compilation, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # return to the main tmp directory os.chdir(tmp) # -------------------------------------------------------------------- # make the runner directory with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nRUNNER STARTS", file=f) tmp_work = os.path.join(tmp,"TMP_WORK") tmp_work_test_input = os.path.join(tmp_work, "test_input") tmp_work_submission = os.path.join(tmp_work, "submitted_files") tmp_work_compiled = os.path.join(tmp_work, "compiled_files") tmp_work_checkout = os.path.join(tmp_work, "checkout") os.mkdir(tmp_work) os.mkdir(tmp_work_test_input) os.mkdir(tmp_work_submission) os.mkdir(tmp_work_compiled) os.mkdir(tmp_work_checkout) os.chdir(tmp_work) # move all executable files from the compilation directory to the main tmp directory # Note: Must preserve the directory structure of compiled files (esp for Java) patterns_submission_to_runner = complete_config_obj["autograding"]["submission_to_runner"] pattern_copy("submission_to_runner",patterns_submission_to_runner,submission_path,tmp_work_submission,tmp_logs) if is_vcs: pattern_copy("checkout_to_runner",patterns_submission_to_runner,checkout_subdir_path,tmp_work_checkout,tmp_logs) # move the compiled files into the tmp_work_compiled directory for path, name in executable_path_list: if not os.path.isfile(path): continue target_path = os.path.join(tmp_work_compiled, name) if not os.path.exists(target_path): os.makedirs(os.path.dirname(target_path), exist_ok=True) shutil.copy(path, target_path) print('copied over {0}'.format(target_path)) with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print('grade_item: copied over {0}'.format(target_path), file=f) patterns_compilation_to_runner = complete_config_obj["autograding"]["compilation_to_runner"] #copy into the actual tmp_work directory for archiving/validating pattern_copy("compilation_to_runner",patterns_compilation_to_runner,tmp_compilation,tmp_work,tmp_logs) #copy into tmp_work_compiled, which is provided to each testcase # TODO change this as our methodology for declaring testcase dependencies becomes more robust pattern_copy("compilation_to_runner",patterns_compilation_to_runner,tmp_compilation,tmp_work_compiled,tmp_logs) # copy input files to tmp_work directory copy_contents_into(job_id,test_input_path,tmp_work_test_input,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy runner.out to the current directory shutil.copy (os.path.join(bin_path,"run.out"),os.path.join(tmp_work,"my_runner.out")) #set the appropriate permissions for the newly created directories #TODO replaces commented out code below add_permissions(os.path.join(tmp_work,"my_runner.out"), stat.S_IXUSR | stat.S_IXGRP |stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_submission, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_compiled, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_checkout, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #TODO this is how permissions used to be set. It was removed because of the way it interacts with the sticky bit. ## give the untrusted user read/write/execute permissions on the tmp directory & files # os.system('ls -al {0}'.format(tmp_work)) # add_permissions_recursive(tmp_work, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) ################################################################################################## #call grade_item_main_runner.py runner_success = grade_item_main_runner.executeTestcases(complete_config_obj, tmp_logs, tmp_work, queue_obj, submission_string, item_name, USE_DOCKER, None, which_untrusted, job_id, grading_began) ################################################################################################## if runner_success == 0: print (which_machine,which_untrusted, "RUNNER OK") else: print (which_machine,which_untrusted, "RUNNER FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="RUNNER FAILURE") add_permissions_recursive(tmp_work, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions_recursive(tmp_compilation, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # -------------------------------------------------------------------- # RUN VALIDATOR with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nVALIDATION STARTS", file=f) # copy results files from compilation... patterns_submission_to_validation = complete_config_obj["autograding"]["submission_to_validation"] pattern_copy("submission_to_validation",patterns_submission_to_validation,submission_path,tmp_work,tmp_logs) if is_vcs: pattern_copy("checkout_to_validation",patterns_submission_to_validation,checkout_subdir_path,tmp_work,tmp_logs) patterns_compilation_to_validation = complete_config_obj["autograding"]["compilation_to_validation"] pattern_copy("compilation_to_validation",patterns_compilation_to_validation,tmp_compilation,tmp_work,tmp_logs) # remove the compilation directory shutil.rmtree(tmp_compilation) # copy output files to tmp_work directory copy_contents_into(job_id,test_output_path,tmp_work,tmp_logs) # copy any instructor custom validation code into the tmp work directory copy_contents_into(job_id,custom_validation_code_path,tmp_work,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy validator.out to the current directory shutil.copy (os.path.join(bin_path,"validate.out"),os.path.join(tmp_work,"my_validator.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(os.path.join(tmp_work,"my_validator.out"), stat.S_IXUSR | stat.S_IXGRP |stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #todo remove prints. print("VALIDATING") # validator the validator.out as the untrusted user with open(os.path.join(tmp_logs,"validator_log.txt"), 'w') as logfile: if USE_DOCKER: # WIP: This option file facilitated testing... #USE_DOCKER = os.path.isfile("/tmp/use_docker") #use_docker_string="grading begins, using DOCKER" if USE_DOCKER else "grading begins (not using docker)" #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message=use_docker_string) container = subprocess.check_output(['docker', 'run', '-t', '-d', '-v', tmp + ':' + tmp, 'ubuntu:custom']).decode('utf8').strip() dockerlaunch_done=dateutils.get_current_time() dockerlaunch_time = (dockerlaunch_done-grading_began).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,"dcct:",dockerlaunch_time,"docker container created") validator_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_validator.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string], stdout=logfile) else: validator_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"sbin","untrusted_execute"), which_untrusted, os.path.join(tmp_work,"my_validator.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string], stdout=logfile) if validator_success == 0: print (which_machine,which_untrusted,"VALIDATOR OK") else: print (which_machine,which_untrusted,"VALIDATOR FAILURE") grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="VALIDATION FAILURE") untrusted_grant_rwx_access(which_untrusted,tmp_work) # grab the result of autograding grade_result = "" try: with open(os.path.join(tmp_work,"grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open ",os.path.join(tmp_work,"grade.txt")) grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="ERROR: grade.txt does not exist") grade_items_logging.log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) # -------------------------------------------------------------------- # MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE tmp_results = os.path.join(tmp,"TMP_RESULTS") with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nARCHIVING STARTS", file=f) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) os.makedirs(os.path.join(tmp_results,"details")) # remove the test_input directory, so we don't archive it! shutil.rmtree(os.path.join(tmp_work,"test_input")) # loop over the test case directories, and remove any files that are also in the test_input folder for testcase_num in range(1, len(my_testcases)+1): testcase_folder = os.path.join(tmp_work, "test{:02}".format(testcase_num)) remove_test_input_files(os.path.join(tmp_logs,"overall.txt"),test_input_path,testcase_folder) patterns_work_to_details = complete_config_obj["autograding"]["work_to_details"] pattern_copy("work_to_details",patterns_work_to_details,tmp_work,os.path.join(tmp_results,"details"),tmp_logs) if ("work_to_public" in complete_config_obj["autograding"] and len(complete_config_obj["autograding"]["work_to_public"]) > 0): # create the directory os.makedirs(os.path.join(tmp_results,"results_public")) # copy the files patterns_work_to_public = complete_config_obj["autograding"]["work_to_public"] pattern_copy("work_to_public",patterns_work_to_public,tmp_work,os.path.join(tmp_results,"results_public"),tmp_logs) history_file_tmp = os.path.join(tmp_submission,"history.json") history_file = os.path.join(tmp_results,"history.json") if os.path.isfile(history_file_tmp): shutil.move(history_file_tmp,history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file,int(DAEMON_UID),ta_group_id) add_permissions(history_file,stat.S_IRGRP) grading_finished = dateutils.get_current_time() try: shutil.copy(os.path.join(tmp_work,"grade.txt"),tmp_results) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not copy ",os.path.join(tmp_work,"grade.txt")) grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="ERROR: grade.txt does not exist") grade_items_logging.log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) # ------------------------------------------------------------- # create/append to the results history # grab the submission time with open (os.path.join(submission_path,".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) gradeable_deadline_datetime = dateutils.read_submitty_date(gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date(gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int((submission_datetime-gradeable_deadline_datetime).total_seconds()) # note: negative = not late grading_finished_longstring = dateutils.write_submitty_date(grading_finished) gradingtime = (grading_finished-grading_began).total_seconds() with open(os.path.join(tmp_submission,"queue_file.json"), 'r') as infile: queue_obj = json.load(infile) queue_obj["gradingtime"]=gradingtime queue_obj["grade_result"]=grade_result queue_obj["which_untrusted"]=which_untrusted with open(os.path.join(tmp_results,"queue_file.json"),'w') as outfile: json.dump(queue_obj,outfile,sort_keys=True,indent=4,separators=(',', ': ')) try: shutil.move(os.path.join(tmp_work, "results.json"), os.path.join(tmp_results, "results.json")) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open/write ",os.path.join(tmp_work,"results.json")) grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="ERROR: results.json read/write error") grade_items_logging.log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) write_grade_history.just_write_grade_history(history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_time_longstring, is_batch_job_string, grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result, revision) os.chdir(SUBMITTY_DATA_DIR) if USE_DOCKER: with open(os.path.join(tmp_logs,"overall_log.txt"), 'w') as logfile: chmod_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container, 'chmod', '-R', 'ugo+rwx', '.'], stdout=logfile) with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: f.write("FINISHED GRADING!\n") # save the logs! shutil.copytree(tmp_logs,os.path.join(tmp_results,"logs")) # zip up results folder filehandle, my_results_zip_file=tempfile.mkstemp() zip_my_directory(tmp_results,my_results_zip_file) os.close(filehandle) shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp_results) shutil.rmtree(tmp_work) shutil.rmtree(tmp) # WIP: extra logging for testing #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message="done grading") # -------------------------------------------------------------------- # CLEAN UP DOCKER if USE_DOCKER: subprocess.call(['docker', 'rm', '-f', container]) dockerdestroy_done=dateutils.get_current_time() dockerdestroy_time = (dockerdestroy_done-grading_finished).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,"ddt:",dockerdestroy_time,"docker container destroyed") grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,"grade:",gradingtime,grade_result) return my_results_zip_file
def worker_process(which_machine,address,which_untrusted,my_server): # verify the DAEMON_USER is running this script if not int(os.getuid()) == int(DAEMON_UID): grade_items_logging.log_message(JOB_ID, message="ERROR: must be run by DAEMON_USER") raise SystemExit("ERROR: the grade_item.py script must be run by the DAEMON_USER") # ignore keyboard interrupts in the worker processes signal.signal(signal.SIGINT, signal.SIG_IGN) counter = 0 servername_workername = "{0}_{1}".format(my_server, address) autograding_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_autograding.zip") submission_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_submission.zip") todo_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_queue.json") while True: if os.path.exists(todo_queue_file): try: results_zip_tmp = grade_item.grade_from_zip(autograding_zip,submission_zip,which_untrusted) results_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_results.zip") done_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_queue.json") #move doesn't inherit the permissions of the destination directory. Copyfile does. shutil.copyfile(results_zip_tmp, results_zip) os.remove(results_zip_tmp) with open(todo_queue_file, 'r') as infile: queue_obj = json.load(infile) queue_obj["done_time"]=dateutils.write_submitty_date(microseconds=True) with open(done_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) except Exception as e: grade_items_logging.log_message(JOB_ID, message="ERROR attempting to unzip graded item: " + which_machine + " " + which_untrusted + ". for more details, see traces entry.") grade_items_logging.log_stack_trace(JOB_ID,trace=traceback.format_exc()) with contextlib.suppress(FileNotFoundError): os.remove(autograding_zip) with contextlib.suppress(FileNotFoundError): os.remove(submission_zip) #Respond with a failure zip file. results_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_results.zip") tmp_dir = tempfile.mkdtemp() with open(os.path.join(tmp_dir, 'failure.txt'), 'w') as outfile: outfile.write("grading failed.\n") results_zip_tmp = zipfile.ZipFile(results_zip, 'w') results_zip_tmp.write(os.path.join(tmp_dir, 'failure.txt')) results_zip_tmp.close() shutil.rmtree(tmp_dir) done_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_queue.json") with open(todo_queue_file, 'r') as infile: queue_obj = json.load(infile) queue_obj["done_time"]=dateutils.write_submitty_date(microseconds=True) with open(done_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) with contextlib.suppress(FileNotFoundError): os.remove(todo_queue_file) counter = 0 else: if counter >= 10: print (which_machine,which_untrusted,"wait") counter = 0 counter += 1 time.sleep(1)
def prepare_autograding_and_submission_zip( which_machine, which_untrusted, next_directory, next_to_grade ): os.chdir(SUBMITTY_DATA_DIR) # generate a random id to be used to track this job in the autograding logs job_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(6)) # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = load_queue_file_obj(job_id, next_directory, next_to_grade) # The top level course directory for this class course_dir = os.path.join(SUBMITTY_DATA_DIR, 'courses', obj["semester"], obj["course"]) if "generate_output" not in obj: partial_path = os.path.join(obj["gradeable"], obj["who"], str(obj["version"])) item_name = os.path.join(obj["semester"], obj["course"], "submissions", partial_path) submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name) if not os.path.isdir(submission_path): autograding_utils.log_message( AUTOGRADING_LOG_PATH, job_id, message=f"ERROR: the submission directory does not exist {submission_path}" ) raise RuntimeError("ERROR: the submission directory does not exist", submission_path) print(which_machine, which_untrusted, "prepare zip", submission_path) is_vcs, vcs_type, vcs_base_url, vcs_subdirectory = get_vcs_info( SUBMITTY_DATA_DIR, obj["semester"], obj["course"], obj["gradeable"], obj["who"], obj["team"] ) elif obj["generate_output"]: item_name = os.path.join( obj["semester"], obj["course"], "generated_output", obj["gradeable"] ) is_batch_job = "regrade" in obj and obj["regrade"] queue_time = get_queue_time(next_directory, next_to_grade) grading_began = dateutils.get_current_time() waittime = (grading_began-queue_time).total_seconds() autograding_utils.log_message( AUTOGRADING_LOG_PATH, job_id, is_batch_job, "zip", item_name, "wait:", waittime, "" ) # -------------------------------------------------------- # various paths provided_code_path = os.path.join(course_dir, "provided_code", obj["gradeable"]) instructor_solution_path = os.path.join(course_dir, "instructor_solution", obj["gradeable"]) test_input_path = os.path.join(course_dir, "test_input", obj["gradeable"]) test_output_path = os.path.join(course_dir, "test_output", obj["gradeable"]) bin_path = os.path.join(course_dir, "bin", obj["gradeable"]) form_json_config = os.path.join(course_dir, "config", "form", f"form_{obj['gradeable']}.json") custom_validation_code_path = os.path.join( course_dir, "custom_validation_code", obj["gradeable"] ) generated_output_path = os.path.join( course_dir, "generated_output", obj["gradeable"], "random_output" ) complete_config = os.path.join( course_dir, "config", "complete_config", f"complete_config_{obj['gradeable']}.json" ) if not os.path.exists(form_json_config): autograding_utils.log_message( AUTOGRADING_LOG_PATH, job_id, message=f"ERROR: the form json file does not exist {form_json_config}" ) raise RuntimeError(f"ERROR: the form json file does not exist {form_json_config}") if not os.path.exists(complete_config): autograding_utils.log_message( AUTOGRADING_LOG_PATH, job_id, message=f"ERROR: the complete config file does not exist {complete_config}" ) raise RuntimeError(f"ERROR: the complete config file does not exist {complete_config}") # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = tempfile.mkdtemp() tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") os.mkdir(tmp_autograding) tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") os.mkdir(tmp_submission) copytree_if_exists(provided_code_path, os.path.join(tmp_autograding, "provided_code")) copytree_if_exists(test_input_path, os.path.join(tmp_autograding, "test_input")) copytree_if_exists(test_output_path, os.path.join(tmp_autograding, "test_output")) copytree_if_exists(generated_output_path, os.path.join(tmp_autograding, "generated_output")) copytree_if_exists(bin_path, os.path.join(tmp_autograding, "bin")) copytree_if_exists( instructor_solution_path, os.path.join(tmp_autograding, "instructor_solution") ) copytree_if_exists( custom_validation_code_path, os.path.join(tmp_autograding, "custom_validation_code") ) # Copy the default submitty_router into bin. router_path = os.path.join( SUBMITTY_INSTALL_DIR, 'src', 'grading', 'python', 'submitty_router.py' ) shutil.copy(router_path, os.path.join(tmp_autograding, "bin")) shutil.copy(form_json_config, os.path.join(tmp_autograding, "form.json")) shutil.copy(complete_config, os.path.join(tmp_autograding, "complete_config.json")) if "generate_output" not in obj: checkout_path = os.path.join(course_dir, "checkout", partial_path) results_path = os.path.join(course_dir, "results", partial_path) elif obj["generate_output"]: results_path = os.path.join(course_dir, "generated_output", obj["gradeable"]) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path, "history.json") if os.path.isfile(history_file): shutil.copy(history_file, os.path.join(tmp_submission, "history.json")) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") os.makedirs(tmp_logs) # 'touch' a file in the logs folder open(os.path.join(tmp_logs, "overall.txt"), 'a') # -------------------------------------------------------------------- # CONFIRM WE HAVE A CHECKOUT OF THE STUDENT'S REPO if "generate_output" not in obj: if is_vcs: # there should be a checkout log file in the results directory # move that file to the tmp logs directory. vcs_checkout_logfile = os.path.join(results_path, "logs", "vcs_checkout.txt") if os.path.isfile(vcs_checkout_logfile): shutil.move(vcs_checkout_logfile, tmp_logs) else: autograding_utils.log_message( AUTOGRADING_LOG_PATH, job_id, message=f"ERROR: missing vcs_checkout.txt logfile {str(vcs_checkout_logfile)}" ) if "generate_output" not in obj: copytree_if_exists(submission_path, os.path.join(tmp_submission, "submission")) copytree_if_exists(checkout_path, os.path.join(tmp_submission, "checkout")) obj["queue_time"] = dateutils.write_submitty_date(queue_time) obj["regrade"] = is_batch_job obj["waittime"] = waittime obj["job_id"] = job_id with open(os.path.join(tmp_submission, "queue_file.json"), 'w') as outfile: json.dump(obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) user_assignment_access_json = os.path.join( SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "submissions", obj["gradeable"], obj["who"], "user_assignment_access.json") user_assignment_settings_json = os.path.join( SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "submissions", obj["gradeable"], obj["who"], "user_assignment_settings.json") if os.path.exists(user_assignment_access_json): shutil.copy( user_assignment_access_json, os.path.join(tmp_submission, "user_assignment_access.json") ) if os.path.exists(user_assignment_settings_json): shutil.copy( user_assignment_settings_json, os.path.join(tmp_submission, "user_assignment_settings.json") ) grading_began_longstring = dateutils.write_submitty_date(grading_began) with open(os.path.join(tmp_submission, ".grading_began"), 'w') as f: print(grading_began_longstring, file=f) # zip up autograding & submission folders filehandle1, my_autograding_zip_file = tempfile.mkstemp() filehandle2, my_submission_zip_file = tempfile.mkstemp() autograding_utils.zip_my_directory(tmp_autograding, my_autograding_zip_file) autograding_utils.zip_my_directory(tmp_submission, my_submission_zip_file) os.close(filehandle1) os.close(filehandle2) # cleanup shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp) return (my_autograding_zip_file, my_submission_zip_file)
def prepare_autograding_and_submission_zip(next_directory, next_to_grade): os.chdir(SUBMITTY_DATA_DIR) # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = load_queue_file_obj(next_directory, next_to_grade) partial_path = os.path.join(obj["gradeable"], obj["who"], str(obj["version"])) item_name = os.path.join(obj["semester"], obj["course"], "submissions", partial_path) submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name) if not os.path.isdir(submission_path): grade_items_logging.log_message( message="ERROR: the submission directory does not exist" + submission_path) raise RuntimeError("ERROR: the submission directory does not exist", submission_path) print("pid", os.getpid(), "GRADE THIS", submission_path) is_vcs, vcs_type, vcs_base_url, vcs_subdirectory = get_vcs_info( SUBMITTY_DATA_DIR, obj["semester"], obj["course"], obj["gradeable"], obj["who"], obj["team"]) is_batch_job = next_directory == BATCH_QUEUE is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory, next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = (grading_began - queue_time).total_seconds() grade_items_logging.log_message(is_batch_job, "zip", item_name, "wait:", waittime, "") # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = tempfile.mkdtemp() tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") os.mkdir(tmp_autograding) tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") os.mkdir(tmp_submission) # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "provided_code", obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_input", obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_output", obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "custom_validation_code", obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "bin", obj["gradeable"]) form_json_config = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "form", "form_" + obj["gradeable"] + ".json") complete_config = os.path.join( SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "complete_config", "complete_config_" + obj["gradeable"] + ".json") copytree_if_exists(provided_code_path, os.path.join(tmp_autograding, "provided_code")) copytree_if_exists(test_input_path, os.path.join(tmp_autograding, "test_input")) copytree_if_exists(test_output_path, os.path.join(tmp_autograding, "test_output")) copytree_if_exists(custom_validation_code_path, os.path.join(tmp_autograding, "custom_validation_code")) copytree_if_exists(bin_path, os.path.join(tmp_autograding, "bin")) shutil.copy(form_json_config, os.path.join(tmp_autograding, "form.json")) shutil.copy(complete_config, os.path.join(tmp_autograding, "complete_config.json")) checkout_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "checkout", partial_path) results_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "results", partial_path) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path, "history.json") history_file_tmp = "" if os.path.isfile(history_file): filehandle, history_file_tmp = tempfile.mkstemp() shutil.copy(history_file, history_file_tmp) shutil.copy(history_file, os.path.join(tmp_submission, "history.json")) # get info from the gradeable config file with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) queue_file = os.path.join(next_directory, next_to_grade) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") os.makedirs(tmp_logs) # 'touch' a file in the logs folder open(os.path.join(tmp_logs, "overall.txt"), 'a') # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) # -------------------------------------------------------------------- # CHECKOUT THE STUDENT's REPO if is_vcs: # is vcs_subdirectory standalone or should it be combined with base_url? if vcs_subdirectory[0] == '/' or '://' in vcs_subdirectory: vcs_path = vcs_subdirectory else: if '://' in vcs_base_url: vcs_path = urllib.parse.urljoin(vcs_base_url, vcs_subdirectory) else: vcs_path = os.path.join(vcs_base_url, vcs_subdirectory) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nVCS CHECKOUT", file=f) print('vcs_base_url', vcs_base_url, file=f) print('vcs_subdirectory', vcs_subdirectory, file=f) print('vcs_path', vcs_path, file=f) print(['/usr/bin/git', 'clone', vcs_path, checkout_path], file=f) # cleanup the previous checkout (if it exists) shutil.rmtree(checkout_path, ignore_errors=True) os.makedirs(checkout_path, exist_ok=True) subprocess.call(['/usr/bin/git', 'clone', vcs_path, checkout_path]) os.chdir(checkout_path) # determine which version we need to checkout what_version = subprocess.check_output([ 'git', 'rev-list', '-n', '1', '--before="' + submission_string + '"', 'master' ]) what_version = str(what_version.decode('utf-8')).rstrip() if what_version == "": # oops, pressed the grade button before a valid commit shutil.rmtree(checkout_path, ignore_errors=True) else: # and check out the right version subprocess.call(['git', 'checkout', '-b', 'grade', what_version]) os.chdir(tmp) subprocess.call(['ls', '-lR', checkout_path], stdout=open(tmp_logs + "/overall.txt", 'a')) copytree_if_exists(submission_path, os.path.join(tmp_submission, "submission")) copytree_if_exists(checkout_path, os.path.join(tmp_submission, "checkout")) obj["queue_time"] = queue_time_longstring obj["is_batch_job"] = is_batch_job obj["waittime"] = waittime with open(os.path.join(tmp_submission, "queue_file.json"), 'w') as outfile: json.dump(obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) grading_began_longstring = dateutils.write_submitty_date(grading_began) with open(os.path.join(tmp_submission, ".grading_began"), 'w') as f: print(grading_began_longstring, file=f) # zip up autograding & submission folders my_autograding_zip_file = tempfile.mkstemp()[1] my_submission_zip_file = tempfile.mkstemp()[1] zip_my_directory(tmp_autograding, my_autograding_zip_file) zip_my_directory(tmp_submission, my_submission_zip_file) # cleanup shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp) return (my_autograding_zip_file, my_submission_zip_file)
def archive_autograding_results(working_directory, job_id, which_untrusted, is_batch_job, complete_config_obj, gradeable_config_obj, queue_obj, log_path, stack_trace_log_path, is_test_environment): """ After grading is finished, archive the results. """ tmp_autograding = os.path.join(working_directory,"TMP_AUTOGRADING") tmp_submission = os.path.join(working_directory,"TMP_SUBMISSION") tmp_work = os.path.join(working_directory,"TMP_WORK") tmp_logs = os.path.join(working_directory,"TMP_SUBMISSION","tmp_logs") tmp_results = os.path.join(working_directory,"TMP_RESULTS") submission_path = os.path.join(tmp_submission, "submission") random_output_path = os.path.join(tmp_work, 'random_output') partial_path = os.path.join(queue_obj["gradeable"],queue_obj["who"],str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"],queue_obj["course"],"submissions",partial_path) results_public_dir = os.path.join(tmp_results,"results_public") results_details_dir = os.path.join(tmp_results, "details") patterns = complete_config_obj['autograding'] # Copy work to details pattern_copy("work_to_details", patterns['work_to_details'], tmp_work, results_details_dir, tmp_logs) # Copy work to public if 'work_to_public' in patterns: pattern_copy("work_to_public", patterns['work_to_public'], tmp_work, results_public_dir, tmp_logs) if os.path.exists(random_output_path): pattern_copy("work_to_random_output", [os.path.join(random_output_path, 'test*', '**', '*.txt'),], tmp_work, tmp_results, tmp_logs) # grab the submission time with open(os.path.join(tmp_submission, 'submission' ,".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() history_file_tmp = os.path.join(tmp_submission,"history.json") history_file = os.path.join(tmp_results,"history.json") if os.path.isfile(history_file_tmp) and not is_test_environment: from . import CONFIG_PATH with open(os.path.join(CONFIG_PATH, 'submitty_users.json')) as open_file: OPEN_JSON = json.load(open_file) DAEMON_UID = OPEN_JSON['daemon_uid'] shutil.move(history_file_tmp, history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file, int(DAEMON_UID),ta_group_id) add_permissions(history_file, stat.S_IRGRP) grading_finished = dateutils.get_current_time() try: shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not copy ",os.path.join(tmp_work,"grade.txt")) log_message(log_path, job_id, is_batch_job, which_untrusted, item_name, message="ERROR: grade.txt does not exist") log_stack_trace(stack_trace_log_path, job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) grade_result = "" try: with open(os.path.join(tmp_work,"grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open ",os.path.join(tmp_work,"grade.txt")) log_message(job_id,is_batch_job,which_untrusted,item_name,message="ERROR: grade.txt does not exist") log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) gradeable_deadline_string = gradeable_config_obj["date_due"] submission_datetime = dateutils.read_submitty_date(submission_string) gradeable_deadline_datetime = dateutils.read_submitty_date(gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date(gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int((submission_datetime-gradeable_deadline_datetime).total_seconds()) # note: negative = not late grading_finished_longstring = dateutils.write_submitty_date(grading_finished) with open(os.path.join(tmp_submission,".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) gradingtime = (grading_finished - grading_began).total_seconds() queue_obj["gradingtime"]=gradingtime queue_obj["grade_result"]=grade_result queue_obj["which_untrusted"]=which_untrusted waittime = queue_obj["waittime"] with open(os.path.join(tmp_results,"queue_file.json"),'w') as outfile: json.dump(queue_obj,outfile,sort_keys=True,indent=4,separators=(',', ': ')) try: shutil.move(os.path.join(tmp_work, "results.json"), os.path.join(tmp_results, "results.json")) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open/write ",os.path.join(tmp_work,"results.json")) log_message(log_path, job_id,is_batch_job,which_untrusted,item_name,message="ERROR: results.json read/write error") log_stack_trace(stack_trace_log_path, job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) just_write_grade_history(history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_obj["queue_time"], "BATCH" if is_batch_job else "INTERACTIVE", grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result, queue_obj.get("revision", None)) with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: f.write("FINISHED GRADING!\n") # save the logs! shutil.copytree(tmp_logs,os.path.join(tmp_results,"logs")) log_message(log_path, job_id,is_batch_job,which_untrusted,item_name,"grade:",gradingtime,grade_result)
def just_grade_item(next_directory,next_to_grade,which_untrusted): my_pid = os.getpid() # verify the hwcron user is running this script if not int(os.getuid()) == int(HWCRON_UID): grade_items_logging.log_message("ERROR: must be run by hwcron") raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user") # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = get_submission_path(next_directory,next_to_grade) submission_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"], "submissions",obj["gradeable"],obj["who"],str(obj["version"])) if not os.path.isdir(submission_path): grade_items_logging.log_message("ERROR: the submission directory does not exist" + submission_path) raise SystemExit("ERROR: the submission directory does not exist",submission_path) print ("pid",my_pid,"GRADE THIS", submission_path) is_vcs,vcs_type,vcs_base_url,vcs_subdirectory = get_vcs_info(SUBMITTY_DATA_DIR,obj["semester"],obj["course"],obj["gradeable"],obj["who"],obj["team"]) is_batch_job = next_directory==BATCH_QUEUE is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory,next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = int((grading_began-queue_time).total_seconds()) grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"wait:",waittime,"") # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"provided_code",obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_input",obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_output",obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"custom_validation_code",obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"bin") checkout_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"checkout",obj["gradeable"],obj["who"],str(obj["version"])) results_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"results",obj["gradeable"],obj["who"],str(obj["version"])) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path,"history.json") history_file_tmp = "" if os.path.isfile(history_file): filehandle,history_file_tmp = tempfile.mkstemp() shutil.copy(history_file,history_file_tmp) # get info from the gradeable config file json_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","form","form_"+obj["gradeable"]+".json") with open(json_config, 'r') as infile: gradeable_config_obj = json.load(infile) # get info from the gradeable config file complete_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","complete_config","complete_config_"+obj["gradeable"]+".json") with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get("use_checkout_subdirectory","") checkout_subdir_path = os.path.join(checkout_path,checkout_subdirectory) # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = os.path.join("/var/local/submitty/autograding_tmp/",which_untrusted,"tmp") shutil.rmtree(tmp,ignore_errors=True) os.makedirs(tmp) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp,"tmp_logs") os.makedirs(tmp_logs) # grab the submission time with open (os.path.join(submission_path,".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) # -------------------------------------------------------------------- # CHECKOUT THE STUDENT's REPO if is_vcs: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nVCS CHECKOUT", file=f) print ("vcs_subdirectory",vcs_subdirectory, file=f) # cleanup the previous checkout (if it exists) shutil.rmtree(checkout_path,ignore_errors=True) os.makedirs(checkout_path, exist_ok=True) subprocess.call (['/usr/bin/git', 'clone', vcs_subdirectory, checkout_path]) os.chdir(checkout_path) # determine which version we need to checkout what_version = subprocess.check_output(['git', 'rev-list', '-n', '1', '--before="'+submission_string+'"', 'master']) what_version = str(what_version.decode('utf-8')).rstrip() if what_version == "": # oops, pressed the grade button before a valid commit shutil.rmtree(checkout_path,ignore_errors=True) else: # and check out the right version subprocess.call (['git', 'checkout', '-b', 'grade', what_version]) os.chdir(tmp) subprocess.call(['ls', '-lR', checkout_path], stdout=open(tmp_logs + "/overall.txt", 'a')) # -------------------------------------------------------------------- # START DOCKER container = None if USE_DOCKER: container = subprocess.check_output(['docker', 'run', '-t', '-d', '-v', tmp + ':' + tmp, 'ubuntu:custom']).decode('utf8').strip() # -------------------------------------------------------------------- # COMPILE THE SUBMITTED CODE with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nCOMPILATION STARTS", file=f) # copy submitted files to the tmp compilation directory tmp_compilation = os.path.join(tmp,"TMP_COMPILATION") os.mkdir(tmp_compilation) os.chdir(tmp_compilation) gradeable_deadline_string = gradeable_config_obj["date_due"] patterns_submission_to_compilation = complete_config_obj["autograding"]["submission_to_compilation"] pattern_copy("submission_to_compilation",patterns_submission_to_compilation,submission_path,tmp_compilation,tmp_logs) if is_vcs: pattern_copy("checkout_to_compilation",patterns_submission_to_compilation,checkout_subdir_path,tmp_compilation,tmp_logs) # copy any instructor provided code files to tmp compilation directory copy_contents_into(provided_code_path,tmp_compilation,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy compile.out to the current directory shutil.copy (os.path.join(bin_path,obj["gradeable"],"compile.out"),os.path.join(tmp_compilation,"my_compile.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_compilation, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP) add_permissions(tmp,stat.S_IROTH | stat.S_IXOTH) add_permissions(tmp_logs,stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) with open(os.path.join(tmp_logs,"compilation_log.txt"), 'w') as logfile: if USE_DOCKER: compile_success = subprocess.call(['docker', 'exec', '-w', tmp_compilation, container, os.path.join(tmp_compilation, 'my_compile.out'), obj['gradeable'], obj['who'], str(obj['version']), submission_string], stdout=logfile) else: compile_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"), which_untrusted, os.path.join(tmp_compilation,"my_compile.out"), obj["gradeable"], obj["who"], str(obj["version"]), submission_string], stdout=logfile) if compile_success == 0: print ("pid",my_pid,"COMPILATION OK") else: print ("pid",my_pid,"COMPILATION FAILURE") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","COMPILATION FAILURE") #raise SystemExit() untrusted_grant_rwx_access(which_untrusted,tmp_compilation) # remove the compilation program os.remove(os.path.join(tmp_compilation,"my_compile.out")) # return to the main tmp directory os.chdir(tmp) # -------------------------------------------------------------------- # make the runner directory with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nRUNNER STARTS", file=f) tmp_work = os.path.join(tmp,"TMP_WORK") os.makedirs(tmp_work) os.chdir(tmp_work) # move all executable files from the compilation directory to the main tmp directory # Note: Must preserve the directory structure of compiled files (esp for Java) patterns_submission_to_runner = complete_config_obj["autograding"]["submission_to_runner"] pattern_copy("submission_to_runner",patterns_submission_to_runner,submission_path,tmp_work,tmp_logs) if is_vcs: pattern_copy("checkout_to_runner",patterns_submission_to_runner,checkout_subdir_path,tmp_work,tmp_logs) patterns_compilation_to_runner = complete_config_obj["autograding"]["compilation_to_runner"] pattern_copy("compilation_to_runner",patterns_compilation_to_runner,tmp_compilation,tmp_work,tmp_logs) # copy input files to tmp_work directory copy_contents_into(test_input_path,tmp_work,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy runner.out to the current directory shutil.copy (os.path.join(bin_path,obj["gradeable"],"run.out"),os.path.join(tmp_work,"my_runner.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # raise SystemExit() # run the run.out as the untrusted user with open(os.path.join(tmp_logs,"runner_log.txt"), 'w') as logfile: if USE_DOCKER: runner_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_runner.out'), obj['gradeable'], obj['who'], str(obj['version']), submission_string], stdout=logfile) else: runner_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"), which_untrusted, os.path.join(tmp_work,"my_runner.out"), obj["gradeable"], obj["who"], str(obj["version"]), submission_string], stdout=logfile) if runner_success == 0: print ("pid",my_pid,"RUNNER OK") else: print ("pid",my_pid,"RUNNER FAILURE") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","RUNNER FAILURE") untrusted_grant_rwx_access(which_untrusted,tmp_work) untrusted_grant_rwx_access(which_untrusted,tmp_compilation) # -------------------------------------------------------------------- # RUN VALIDATOR with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nVALIDATION STARTS", file=f) # copy results files from compilation... patterns_submission_to_validation = complete_config_obj["autograding"]["submission_to_validation"] pattern_copy("submission_to_validation",patterns_submission_to_validation,submission_path,tmp_work,tmp_logs) if is_vcs: pattern_copy("checkout_to_validation",patterns_submission_to_validation,checkout_subdir_path,tmp_work,tmp_logs) patterns_compilation_to_validation = complete_config_obj["autograding"]["compilation_to_validation"] pattern_copy("compilation_to_validation",patterns_compilation_to_validation,tmp_compilation,tmp_work,tmp_logs) # remove the compilation directory shutil.rmtree(tmp_compilation) # copy output files to tmp_work directory copy_contents_into(test_output_path,tmp_work,tmp_logs) # copy any instructor custom validation code into the tmp work directory copy_contents_into(custom_validation_code_path,tmp_work,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy validator.out to the current directory shutil.copy (os.path.join(bin_path,obj["gradeable"],"validate.out"),os.path.join(tmp_work,"my_validator.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(os.path.join(tmp_work,"my_validator.out"),stat.S_IROTH | stat.S_IXOTH) # validator the validator.out as the untrusted user with open(os.path.join(tmp_logs,"validator_log.txt"), 'w') as logfile: if USE_DOCKER: validator_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_validator.out'), obj['gradeable'], obj['who'], str(obj['version']), submission_string], stdout=logfile) else: validator_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"), which_untrusted, os.path.join(tmp_work,"my_validator.out"), obj["gradeable"], obj["who"], str(obj["version"]), submission_string], stdout=logfile) if validator_success == 0: print ("pid",my_pid,"VALIDATOR OK") else: print ("pid",my_pid,"VALIDATOR FAILURE") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","VALIDATION FAILURE") untrusted_grant_rwx_access(which_untrusted,tmp_work) # grab the result of autograding grade_result = "" with open(os.path.join(tmp_work,"grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line # -------------------------------------------------------------------- # MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nARCHIVING STARTS", file=f) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) os.chdir(bin_path) # save the old results path! if os.path.isdir(os.path.join(results_path,"OLD")): shutil.move(os.path.join(results_path,"OLD"), os.path.join(tmp,"OLD_RESULTS")) # clean out all of the old files if this is a re-run shutil.rmtree(results_path,ignore_errors=True) # create the directory (and the full path if it doesn't already exist) os.makedirs(results_path) # bring back the old results! if os.path.isdir(os.path.join(tmp,"OLD_RESULTS")): shutil.move(os.path.join(tmp,"OLD_RESULTS"), os.path.join(results_path,"OLD")) os.makedirs(os.path.join(results_path,"details")) patterns_work_to_details = complete_config_obj["autograding"]["work_to_details"] pattern_copy("work_to_details",patterns_work_to_details,tmp_work,os.path.join(results_path,"details"),tmp_logs) if not history_file_tmp == "": shutil.move(history_file_tmp,history_file) # fix permissions ta_group_id = os.stat(results_path).st_gid os.chown(history_file,int(HWCRON_UID),ta_group_id) add_permissions(history_file,stat.S_IRGRP) grading_finished = dateutils.get_current_time() shutil.copy(os.path.join(tmp_work,"results.json"),results_path) shutil.copy(os.path.join(tmp_work,"grade.txt"),results_path) # ------------------------------------------------------------- # create/append to the results history gradeable_deadline_datetime = dateutils.read_submitty_date(gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date(gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int((submission_datetime-gradeable_deadline_datetime).total_seconds()) # note: negative = not late grading_began_longstring = dateutils.write_submitty_date(grading_began) grading_finished_longstring = dateutils.write_submitty_date(grading_finished) gradingtime = int((grading_finished-grading_began).total_seconds()) write_grade_history.just_write_grade_history(history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_time_longstring, is_batch_job_string, grading_began_longstring, waittime, grading_finished_longstring, gradingtime, grade_result) #--------------------------------------------------------------------- # WRITE OUT VERSION DETAILS if WRITE_DATABASE: insert_database_version_data.insert_to_database( obj["semester"], obj["course"], obj["gradeable"], obj["user"], obj["team"], obj["who"], True if obj["is_team"] else False, str(obj["version"])) print ("pid",my_pid,"finished grading ", next_to_grade, " in ", gradingtime, " seconds") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"grade:",gradingtime,grade_result) with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: f.write("FINISHED GRADING!") # save the logs! shutil.copytree(tmp_logs,os.path.join(results_path,"logs")) # -------------------------------------------------------------------- # REMOVE TEMP DIRECTORY shutil.rmtree(tmp) # -------------------------------------------------------------------- # CLEAN UP DOCKER if USE_DOCKER: subprocess.call(['docker', 'rm', '-f', container])