def clean_up_containers(container_info,job_id,is_batch_job,which_untrusted,submission_path,grading_began,use_router): # First, clean up the dockers. for name, info in container_info.items(): c_id = info['container_id'] subprocess.call(['docker', 'rm', '-f', c_id]) dockerdestroy_done=dateutils.get_current_time() dockerdestroy_time = (dockerdestroy_done-grading_began).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,"ddt:", dockerdestroy_time, "docker container {0} destroyed".format(name)) if not use_router: network_name = '{0}_routerless_network'.format(which_untrusted) subprocess.call(['docker', 'network', 'rm', network_name]) network_destroy_done=dateutils.get_current_time() network_destroy_time = (network_destroy_done-grading_began).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,"ddt:", network_destroy_time,"docker network {0} destroyed".format(network_name)) else: #Networks must be removed AFTER all docker endpoints have been shut down. for name, info in container_info.items(): if 'network' in info: network_name = info['network'] subprocess.call(['docker', 'network', 'rm', network_name]) network_destroy_done=dateutils.get_current_time() network_destroy_time = (network_destroy_done-grading_began).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,"ddt:", network_destroy_time,"docker network {0} destroyed".format(network_name))
def clean_up_containers(container_info,job_id,is_batch_job,which_untrusted,submission_path,grading_began,use_router): # First, clean up the dockers. for name, info in container_info.items(): if not "container_id" in info: continue c_id = info['container_id'] subprocess.call(['docker', 'rm', '-f', c_id]) dockerdestroy_done=dateutils.get_current_time() dockerdestroy_time = (dockerdestroy_done-grading_began).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,"ddt:", dockerdestroy_time, "docker container {0} destroyed".format(name)) if not use_router: network_name = '{0}_routerless_network'.format(which_untrusted) subprocess.call(['docker', 'network', 'rm', network_name]) network_destroy_done=dateutils.get_current_time() network_destroy_time = (network_destroy_done-grading_began).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,"ddt:", network_destroy_time,"docker network {0} destroyed".format(network_name)) else: #Networks must be removed AFTER all docker endpoints have been shut down. for name, info in container_info.items(): if 'network' in info: network_name = info['network'] subprocess.call(['docker', 'network', 'rm', network_name]) network_destroy_done=dateutils.get_current_time() network_destroy_time = (network_destroy_done-grading_began).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,"ddt:", network_destroy_time,"docker network {0} destroyed".format(network_name))
def create(self, execution_script, arguments, more_than_one): """ Create (but don't start) this container. """ # Only pass container name to testcases with greater than one container. (Doing otherwise breaks compilation) conatiner_name_argument = ['--container_name', self.name ] if more_than_one else list() # A server container does not run student code, but instead hosts a service (e.g. a database.) if self.is_server: container_id = subprocess.check_output([ 'docker', 'create', '-i', '--network', 'none', '-v', f'{self.directory}:{self.directory}', '-w', self.directory, '--name', self.full_name, self.image ]).decode('utf8').strip() else: this_container = subprocess.check_output( ['docker', 'create', '-i', '--network', 'none'] + self.container_user_argument + [ '-v', self.directory + ':' + self.directory, '-w', self.directory, '--hostname', self.name, '--name', self.full_name, self.image, execution_script ] + arguments + conatiner_name_argument).decode('utf8').strip() dockerlaunch_done = dateutils.get_current_time() self.log_function(f'docker container {this_container} created') self.container_id = this_container
def log_message(log_path, job_id="UNKNOWN", is_batch=False, which_untrusted="", jobname="", timelabel="", elapsed_time=-1, message=""): """ Given a log directory, create or append a message to a dated log file in that directory. """ now = dateutils.get_current_time() datefile = datetime.strftime(now, "%Y%m%d") + ".txt" autograding_log_file = os.path.join(log_path, datefile) easy_to_read_date = dateutils.write_submitty_date(now, True) batch_string = "BATCH" if is_batch else "" if elapsed_time == "": elapsed_time = -1 elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format( elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" with open(autograding_log_file, 'a') as myfile: try: fcntl.flock(myfile, fcntl.LOCK_EX | fcntl.LOCK_NB) print( "%s | %6s | %5s | %11s | %-75s | %-6s %9s %3s | %s" % (easy_to_read_date, job_id, batch_string, which_untrusted, jobname, timelabel, elapsed_time_string, time_unit, message), file=myfile) fcntl.flock(myfile, fcntl.LOCK_UN) except: print("Could not gain a lock on the log file.")
def log_message(log_path, job_id="UNKNOWN", is_batch=False, which_untrusted="", jobname="", timelabel="", elapsed_time=-1, message=""): """ Given a log directory, create or append a message to a dated log file in that directory. """ now = dateutils.get_current_time() datefile = datetime.strftime(now, "%Y%m%d") + ".txt" autograding_log_file = os.path.join(log_path, datefile) easy_to_read_date = dateutils.write_submitty_date(now, True) batch_string = "BATCH" if is_batch else "" if elapsed_time == "": elapsed_time = -1 elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format( elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" parts = (easy_to_read_date, f"{job_id:>6s}", f"{batch_string:>5s}", f"{which_untrusted:>11s}", f"{jobname:75s}", f"{timelabel:6s} {elapsed_time_string:>9s} {time_unit:>3s}", message) write_to_log(autograding_log_file, parts)
def _log_filename(self) -> str: """Get the name of the file that should be logged into. Currently, this is in the format YYYYMMDD.txt. """ now = dateutils.get_current_time() return f'{datetime.strftime(now, "%Y%m%d")}.txt'
def log_stack_trace(job_id="UNKNOWN", is_batch=False, which_untrusted="", jobname="", timelabel="", elapsed_time=-1, trace=""): now = dateutils.get_current_time() datefile = "stack_traces_{0}.txt".format(datetime.strftime(now, "%Y%m%d")) autograding_log_file = os.path.join(AUTOGRADING_STACKTRACE_PATH, datefile) easy_to_read_date = dateutils.write_submitty_date(now, True) batch_string = "BATCH" if is_batch else "" if elapsed_time == "": elapsed_time = -1 elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format( elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" with open(autograding_log_file, 'a') as myfile: try: fcntl.flock(myfile, fcntl.LOCK_EX | fcntl.LOCK_NB) print("%s | %6s | %5s | %11s | %-75s | %-6s %9s %3s |\n%s" % (easy_to_read_date, job_id, batch_string, which_untrusted, jobname, timelabel, elapsed_time_string, time_unit, trace), file=myfile) fcntl.flock(myfile, fcntl.LOCK_UN) except: print("Could not gain a lock on the log file.")
def create(self, execution_script, arguments, more_than_one): """ Create (but don't start) this container. """ client = docker.from_env() mount = { self.directory : { 'bind' : self.directory, 'mode' : 'rw' } } # Only pass container name to testcases with greater than one container. (Doing otherwise breaks compilation) container_name_argument = ['--container_name', self.name] if more_than_one else list() # A server container does not run student code, but instead hosts a service (e.g. a database.) try: if self.is_server: self.container = client.containers.create(self.image, stdin_open = True, tty = True, network = 'none', volumes = mount, working_dir = self.directory, name = self.full_name) else: container_ulimits = rlimit_utils.build_ulimit_argument(self.container_rlimits, self.image) command = [execution_script,] + arguments + container_name_argument self.container = client.containers.create(self.image, command = command, ulimits = container_ulimits, stdin_open = True, tty = True, network = 'none', user = self.container_user_argument, volumes=mount, working_dir = self.directory, hostname = self.name, name = self.full_name) except docker.errors.ImageNotFound: self.log_function(f'ERROR: The image {self.image} is not available on this worker') client.close() raise dockerlaunch_done = dateutils.get_current_time() self.log_function(f'docker container {self.container.short_id} created') client.close()
def create_container(container_name, container_image, server_container, mounted_directory, job_id, is_batch_job, which_untrusted, submission_path, grading_began, queue_obj, submission_string, testcase_num, name): untrusted_uid = str(getpwnam(which_untrusted).pw_uid) if server_container: this_container = subprocess.check_output([ 'docker', 'create', '-i', '--network', 'none', '-v', mounted_directory + ':' + mounted_directory, '-w', mounted_directory, '--name', container_name, container_image ]).decode('utf8').strip() else: this_container = subprocess.check_output([ 'docker', 'create', '-i', '-u', untrusted_uid, '--network', 'none', '-v', mounted_directory + ':' + mounted_directory, '-w', mounted_directory, '--hostname', name, '--name', container_name, container_image, os.path.join(mounted_directory, 'my_runner.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string, str(testcase_num), name ]).decode('utf8').strip() dockerlaunch_done = dateutils.get_current_time() dockerlaunch_time = (dockerlaunch_done - grading_began).total_seconds() grade_items_logging.log_message( job_id, is_batch_job, which_untrusted, submission_path, "dcct:", dockerlaunch_time, "docker container {0} created".format(this_container)) return this_container
def log_container_meta(log_path, event="", name="", container="", time=0): """ Given a log file, create or append container meta data to a log file. """ now = dateutils.get_current_time() easy_to_read_date = dateutils.write_submitty_date(now, True) time_unit = "sec" parts = (easy_to_read_date, name, container, event, f"{time:.3f}", time_unit) write_to_log(log_path, parts)
def logMessage(message): """ Write a message to the submitty system logs folder to aid in debugging. """ now = dateutils.get_current_time() now_filename = datetime.strftime(now, "%Y%m%d") filename = f"/var/local/submitty/logs/daemon_job_queue/{now_filename}.txt" pid = os.getpid() now_format = datetime.strftime(now, "%Y-%m-%d %H:%M:%S") dated_message = f"{now_format} | {pid:>7} | {message}" with open(filename, "a") as logfile: logfile.write(dated_message + "\n") logfile.flush() print(dated_message, flush=True)
def launch_container(container_name, container_image, mounted_directory, job_id, is_batch_job, which_untrusted, submission_path, grading_began): #TODO error handling. this_container = subprocess.check_output([ 'docker', 'run', '-t', '-d', '-v', mounted_directory + ':' + mounted_directory, '--name', container_name, container_image ]).decode('utf8').strip() dockerlaunch_done = dateutils.get_current_time() dockerlaunch_time = (dockerlaunch_done - grading_began).total_seconds() grade_items_logging.log_message( job_id, is_batch_job, which_untrusted, submission_path, "dcct:", dockerlaunch_time, "docker container {0} created".format(this_container)) return this_container
def log_message(is_batch, which_untrusted, jobname, timelabel, elapsed_time, message): now = dateutils.get_current_time() datefile = datetime.strftime(now, "%Y%m%d") + ".txt" autograding_log_file = os.path.join(AUTOGRADING_LOG_PATH, datefile) easy_to_read_date = dateutils.write_submitty_date(now) my_pid = os.getpid() parent_pid = os.getppid() batch_string = "BATCH" if is_batch else "" abbrev_jobname = jobname[len(SUBMITTY_DATA_DIR + "/courses/"):] time_unit = "" if elapsed_time == "" else "sec" with open(autograding_log_file, 'a') as myfile: fcntl.flock(myfile, fcntl.LOCK_EX | fcntl.LOCK_NB) print("%s | %6s | %5s | %11s | %-75s | %-6s %5s %3s | %s" % (easy_to_read_date, my_pid, batch_string, which_untrusted, abbrev_jobname, timelabel, elapsed_time, time_unit, message), file=myfile) fcntl.flock(myfile, fcntl.LOCK_UN)
def generateRandomSuperuserEmail(recipients): with open(os.path.join(SETUP_DATA_PATH, 'random', 'SuperuserEmailBody.txt')) as body_file, \ open(os.path.join(SETUP_DATA_PATH, 'random', 'SuperuserSubject.txt')) as subject_file: body = random.choice(body_file.read().strip().split('\n')) subject = random.choice(subject_file.read().strip().split('\n')) now = dateutils.get_current_time() emails = [] for recipient in recipients: emails.append( { "user_id": recipient.user_id, "subject": "[Submitty Admin Announcement]: " + subject, "body": body, "created": now, "email_address": recipient.user_email, "semester": None, "course": None } ) return emails
def log_stack_trace(job_id="UNKNOWN", is_batch=False, which_untrusted="", jobname="", timelabel="", elapsed_time=-1, trace=""): now = dateutils.get_current_time() datefile = "stack_traces_{0}.txt".format(datetime.strftime(now, "%Y%m%d")) autograding_log_file = os.path.join(AUTOGRADING_STACKTRACE_PATH, datefile) easy_to_read_date = dateutils.write_submitty_date(now, True) batch_string = "BATCH" if is_batch else "" if elapsed_time == "": elapsed_time = -1 elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format(elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" with open(autograding_log_file, 'a') as myfile: try: fcntl.flock(myfile,fcntl.LOCK_EX | fcntl.LOCK_NB) print("%s | %6s | %5s | %11s | %-75s | %-6s %9s %3s |\n%s" % (easy_to_read_date, job_id, batch_string, which_untrusted, jobname, timelabel, elapsed_time_string, time_unit, trace), file=myfile) fcntl.flock(myfile, fcntl.LOCK_UN) except: print("Could not gain a lock on the log file.")
def log_stack_trace( self, trace: str, *, is_batch: bool = False, which_untrusted: str = '', job_id: Optional[str] = None, jobname: str = "", echo_source: Optional[str] = None, ): """Log a stack trace to this logger's configured stack trace directory.""" job_id = job_id or self.job_id # Save the parameters to this trace so we can duplicate these on the # shipper's end once the job finishes. # # TODO: Maybe we want to store time info too? Might need to think a bit # more in terms of the stack traces log file format. if self.capture_traces: self.accumulated_traces.append({ 'trace': trace, 'is_batch': is_batch, 'which_untrusted': which_untrusted, 'job_id': job_id, 'jobname': jobname, }) # Always run this since this could be deleted without us knowing os.makedirs(self.stack_trace_dir, exist_ok=True) now = dateutils.get_current_time() easy_to_read_date = dateutils.write_submitty_date(now, True) message = f"[{easy_to_read_date}][{job_id:>6s}]\n" if echo_source is not None: message += f"== (Echoed from {echo_source})\n" message += f"== Batch? {is_batch}\n" message += f"== Which: {which_untrusted}\n" message += f"== Job: {jobname}\n" for line in trace.splitlines(): message += f"== {line}\n" message = message.strip() write_to_log(self.stack_trace_path, message)
def generateRandomCourseEmail(recipients, course): with open(os.path.join(SETUP_DATA_PATH, 'random', 'CourseEmailBody.txt')) as body_file, \ open(os.path.join(SETUP_DATA_PATH, 'random', 'CourseSubject.txt')) as subject_file: body = random.choice(body_file.read().strip().split('\n')) subject = random.choice(subject_file.read().strip().split('\n')) now = dateutils.get_current_time() emails = [] for recipient in recipients: emails.append( { "user_id": recipient.user_id, "subject": subject, "body": body, "created": now, "email_address": recipient.user_email, "semester": course.semester, "course": course.course } ) return emails
def log_message( self, message: str, *, is_batch: bool = False, which_untrusted: str = "", jobname: str = "", timelabel: str = "", elapsed_time: Optional[int] = None, job_id: Optional[str] = None ): """Log a message to this logger's configured log directory.""" now = dateutils.get_current_time() easy_to_read_date = dateutils.write_submitty_date(now, True) batch_string = "BATCH" if is_batch else "" if elapsed_time is None: elapsed_time = -1 elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format(elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" job_id = job_id or self.job_id parts = (easy_to_read_date, f"{job_id:>6s}", f"{batch_string:>5s}", f"{which_untrusted:>11s}", f"{jobname:75s}", f"{timelabel:6s} {elapsed_time_string:>9s} {time_unit:>3s}", message) write_to_log(self.log_path, ' | '.join((str(x) for x in parts)))
def log_message(is_batch=False, which_untrusted="", jobname="", timelabel="", elapsed_time=-1, message=""): now = dateutils.get_current_time() datefile = datetime.strftime(now, "%Y%m%d") + ".txt" autograding_log_file = os.path.join(AUTOGRADING_LOG_PATH, datefile) easy_to_read_date = dateutils.write_submitty_date(now, True) my_pid = os.getpid() parent_pid = os.getppid() batch_string = "BATCH" if is_batch else "" elapsed_time_string = "" if elapsed_time < 0 else '{:9.3f}'.format( elapsed_time) time_unit = "" if elapsed_time < 0 else "sec" with open(autograding_log_file, 'a') as myfile: fcntl.flock(myfile, fcntl.LOCK_EX | fcntl.LOCK_NB) print("%s | %6s | %5s | %11s | %-75s | %-6s %9s %3s | %s" % (easy_to_read_date, my_pid, batch_string, which_untrusted, jobname, timelabel, elapsed_time_string, time_unit, message), file=myfile) fcntl.flock(myfile, fcntl.LOCK_UN)
def create_container(container_name, container_image, server_container, mounted_directory,job_id, is_batch_job,which_untrusted,submission_path, grading_began,queue_obj,submission_string,testcase_num,name): untrusted_uid = str(getpwnam(which_untrusted).pw_uid) display_sys_variable = str(os.environ.get('DISPLAY', None)) display_line = [] if display_sys_variable == None else ['--display', display_sys_variable] if server_container: this_container = subprocess.check_output(['docker', 'create', '-i', '--network', 'none', '-v', mounted_directory + ':' + mounted_directory, '-w', mounted_directory, '--name', container_name, container_image ]).decode('utf8').strip() else: this_container = subprocess.check_output(['docker', 'create', '-i', '-u', untrusted_uid, '--network', 'none', '-v', mounted_directory + ':' + mounted_directory, '-w', mounted_directory, '--hostname', name, '--name', container_name, container_image, os.path.join(mounted_directory, 'my_runner.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string, '--testcase', str(testcase_num), '--container_name', name] + display_line ).decode('utf8').strip() dockerlaunch_done =dateutils.get_current_time() dockerlaunch_time = (dockerlaunch_done-grading_began).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,"dcct:",dockerlaunch_time, "docker container {0} created".format(this_container)) return this_container
def get_job(my_name,which_machine,my_capabilities,which_untrusted,overall_lock): """ Picks a job from the queue :param overall_lock: a lock on the directory containing all queue files """ time_get_job_begin = dateutils.get_current_time() overall_lock.acquire() folder= INTERACTIVE_QUEUE # ---------------------------------------------------------------- # Our first priority is to perform any awaiting VCS checkouts # Note: This design is imperfect: # # * If all shippers are busy working on long-running autograding # tasks there will be a delay of seconds or minutes between # a student pressing the submission button and clone happening. # This is a minor exploit allowing them to theoretically # continue working on their submission past the deadline for # the time period of the delay. # -- This is not a significant, practical problem. # # * If multiple and/or large git submissions arrive close # together, this shipper job will be tied up performing these # clone operations. Because we don't release the lock, any # other shippers that complete their work will also be blocked # from either helping with the clones or tackling the next # autograding job. # -- Based on experience with actual submission patterns, we # do not anticipate that this will be a significant # bottleneck at this time. # # * If a git clone takes a very long time and/or hangs because of # network problems, this could halt all work on the server. # -- We'll need to monitor the production server. # # We plan to do a complete overhaul of the # scheduler/shipper/worker and refactoring this design should be # part of the project. # Grab all the VCS files currently in the folder... vcs_files = [str(f) for f in Path(folder).glob('VCS__*')] for f in vcs_files: vcs_file = f[len(folder)+1:] no_vcs_file = f[len(folder)+1+5:] # do the checkout updated_obj = checkout_vcs_repo(folder+"/"+vcs_file) # save the regular grading queue file with open(os.path.join(folder,no_vcs_file), "w") as queue_file: json.dump(updated_obj, queue_file) # cleanup the vcs queue file os.remove(folder+"/"+vcs_file) # ---------------------------------------------------------------- # Grab all the files currently in the folder, sorted by creation # time, and put them in the queue to be graded files = [str(f) for f in Path(folder).glob('*')] files_and_times = list() for f in files: try: my_time = os.path.getctime(f) except: continue tup = (f, my_time) files_and_times.append(tup) files_and_times = sorted(files_and_times, key=operator.itemgetter(1)) my_job="" for full_path_file, file_time in files_and_times: # get the file name (without the path) just_file = full_path_file[len(folder)+1:] # skip items that are already being graded if (just_file[0:8]=="GRADING_"): continue grading_file = os.path.join(folder,"GRADING_"+just_file) if grading_file in files: continue # skip items (very recently added!) that are already waiting for a VCS checkout if (just_file[0:5]=="VCS__"): continue # found something to do try: with open(full_path_file, 'r') as infile: queue_obj = json.load(infile) except: continue #Check to make sure that we are capable of grading this submission required_capabilities = queue_obj["required_capabilities"] if not required_capabilities in my_capabilities: continue # prioritize interactive jobs over (batch) regrades # if you've found an interactive job, exit early (since they are sorted by timestamp) if not "regrade" in queue_obj or not queue_obj["regrade"]: my_job = just_file break # otherwise it's a regrade, and if we don't already have a # job, take it, but we have to search the rest of the list if my_job == "": my_job = just_file if not my_job == "": grading_file = os.path.join(folder, "GRADING_" + my_job) # create the grading file with open(os.path.join(grading_file), "w") as queue_file: json.dump({"untrusted": which_untrusted, "machine": which_machine}, queue_file) overall_lock.release() time_get_job_end = dateutils.get_current_time() time_delta = time_get_job_end-time_get_job_begin if time_delta > datetime.timedelta(milliseconds=100): print (my_name, " WARNING: submitty_autograding shipper get_job time ", time_delta) autograding_utils.log_message(AUTOGRADING_LOG_PATH, JOB_ID, message=str(my_name)+" WARNING: submitty_autograding shipper get_job time "+str(time_delta)) return (my_job)
def grade_from_zip(my_autograding_zip_file,my_submission_zip_file,which_untrusted): os.chdir(SUBMITTY_DATA_DIR) tmp = os.path.join("/var/local/submitty/autograding_tmp/",which_untrusted,"tmp") if os.path.exists(tmp): untrusted_grant_rwx_access(which_untrusted, tmp) add_permissions_recursive(tmp, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # Remove any and all containers left over from past runs. old_containers = subprocess.check_output(['docker', 'ps', '-aq', '-f', 'name={0}'.format(which_untrusted)]).split() for old_container in old_containers: subprocess.call(['docker', 'rm', '-f', old_container.decode('utf8')]) # clean up old usage of this directory shutil.rmtree(tmp,ignore_errors=True) os.mkdir(tmp) which_machine=socket.gethostname() # unzip autograding and submission folders tmp_autograding = os.path.join(tmp,"TMP_AUTOGRADING") tmp_submission = os.path.join(tmp,"TMP_SUBMISSION") try: unzip_this_file(my_autograding_zip_file,tmp_autograding) unzip_this_file(my_submission_zip_file,tmp_submission) except: raise os.remove(my_autograding_zip_file) os.remove(my_submission_zip_file) tmp_logs = os.path.join(tmp,"TMP_SUBMISSION","tmp_logs") queue_file = os.path.join(tmp_submission,"queue_file.json") with open(queue_file, 'r') as infile: queue_obj = json.load(infile) queue_time_longstring = queue_obj["queue_time"] waittime = queue_obj["waittime"] is_batch_job = queue_obj["regrade"] job_id = queue_obj["job_id"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" revision = queue_obj.get("revision", None) partial_path = os.path.join(queue_obj["gradeable"],queue_obj["who"],str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"],queue_obj["course"],"submissions",partial_path) grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,"wait:",waittime,"") with open(os.path.join(tmp_submission,".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) submission_path = os.path.join(tmp_submission, "submission") checkout_path = os.path.join(tmp_submission, "checkout") provided_code_path = os.path.join(tmp_autograding, "provided_code") test_input_path = os.path.join(tmp_autograding, "test_input") test_output_path = os.path.join(tmp_autograding, "test_output") custom_validation_code_path = os.path.join(tmp_autograding, "custom_validation_code") bin_path = os.path.join(tmp_autograding, "bin") form_json_config = os.path.join(tmp_autograding, "form.json") complete_config = os.path.join(tmp_autograding, "complete_config.json") with open(form_json_config, 'r') as infile: gradeable_config_obj = json.load(infile) gradeable_deadline_string = gradeable_config_obj["date_due"] with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) is_vcs = gradeable_config_obj["upload_type"] == "repository" checkout_subdirectory = complete_config_obj["autograding"].get("use_checkout_subdirectory","") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) if complete_config_obj.get('one_part_only', False): allow_only_one_part(submission_path, os.path.join(tmp_logs, "overall.txt")) if is_vcs: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("WARNING: ONE_PART_ONLY OPTION DOES NOT MAKE SENSE WITH VCS SUBMISSION", file=f) # -------------------------------------------------------------------- # START DOCKER # NOTE: DOCKER SUPPORT PRELIMINARY -- NEEDS MORE SECURITY BEFORE DEPLOYED ON LIVE SERVER complete_config = os.path.join(tmp_autograding,"complete_config.json") with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) # Save ourselves if autograding_method is None. autograding_method = complete_config_obj.get("autograding_method", "") USE_DOCKER = True if autograding_method == "docker" else False # -------------------------------------------------------------------- # COMPILE THE SUBMITTED CODE with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nCOMPILATION STARTS", file=f) # copy submitted files to the tmp compilation directory tmp_compilation = os.path.join(tmp,"TMP_COMPILATION") os.mkdir(tmp_compilation) os.chdir(tmp_compilation) submission_path = os.path.join(tmp_submission,"submission") checkout_path = os.path.join(tmp_submission,"checkout") provided_code_path = os.path.join(tmp_autograding,"provided_code") test_input_path = os.path.join(tmp_autograding,"test_input") test_output_path = os.path.join(tmp_autograding,"test_output") custom_validation_code_path = os.path.join(tmp_autograding,"custom_validation_code") bin_path = os.path.join(tmp_autograding,"bin") form_json_config = os.path.join(tmp_autograding,"form.json") with open(form_json_config, 'r') as infile: gradeable_config_obj = json.load(infile) gradeable_deadline_string = gradeable_config_obj["date_due"] patterns_submission_to_compilation = complete_config_obj["autograding"]["submission_to_compilation"] add_permissions(tmp_logs,stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) if USE_DOCKER: print("!!!!!!!!!!!!!!!!!!USING DOCKER!!!!!!!!!!!!!!!!!!!!!!!!") with open(complete_config, 'r') as infile: config = json.load(infile) my_testcases = config['testcases'] # grab the submission time with open(os.path.join(submission_path,".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() with open(os.path.join(tmp_logs,"compilation_log.txt"), 'w') as logfile: # we start counting from one. executable_path_list = list() for testcase_num in range(1, len(my_testcases)+1): testcase_folder = os.path.join(tmp_compilation, "test{:02}".format(testcase_num)) if 'type' in my_testcases[testcase_num-1]: if my_testcases[testcase_num-1]['type'] != 'FileCheck' and my_testcases[testcase_num-1]['type'] != 'Compilation': continue if my_testcases[testcase_num-1]['type'] == 'Compilation': if 'executable_name' in my_testcases[testcase_num-1]: provided_executable_list = my_testcases[testcase_num-1]['executable_name'] if not isinstance(provided_executable_list, (list,)): provided_executable_list = list([provided_executable_list]) for executable_name in provided_executable_list: if executable_name.strip() == '': continue executable_path = os.path.join(testcase_folder, executable_name) executable_path_list.append((executable_path, executable_name)) else: continue os.makedirs(testcase_folder) pattern_copy("submission_to_compilation",patterns_submission_to_compilation,submission_path,testcase_folder,tmp_logs) if is_vcs: pattern_copy("checkout_to_compilation",patterns_submission_to_compilation,checkout_subdir_path,testcase_folder,tmp_logs) # copy any instructor provided code files to tmp compilation directory copy_contents_into(job_id,provided_code_path,testcase_folder,tmp_logs) # copy compile.out to the current directory shutil.copy (os.path.join(bin_path,"compile.out"),os.path.join(testcase_folder,"my_compile.out")) add_permissions(os.path.join(testcase_folder,"my_compile.out"), stat.S_IXUSR | stat.S_IXGRP |stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #untrusted_grant_rwx_access(which_untrusted, tmp_compilation) untrusted_grant_rwx_access(which_untrusted, testcase_folder) add_permissions_recursive(testcase_folder, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) if USE_DOCKER: try: #There can be only one container for a compilation step, so grab its container image #TODO: set default in load_config_json.cpp if my_testcases[testcase_num-1]['type'] == 'FileCheck': print("performing filecheck in default ubuntu:custom container") container_image = "ubuntu:custom" else: container_image = my_testcases[testcase_num-1]["containers"][0]["container_image"] print('creating a compilation container with image {0}'.format(container_image)) untrusted_uid = str(getpwnam(which_untrusted).pw_uid) compilation_container = None compilation_container = subprocess.check_output(['docker', 'create', '-i', '-u', untrusted_uid, '--network', 'none', '-v', testcase_folder + ':' + testcase_folder, '-w', testcase_folder, container_image, #The command to be run. os.path.join(testcase_folder, 'my_compile.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string, '--testcase', str(testcase_num) ]).decode('utf8').strip() print("starting container") compile_success = subprocess.call(['docker', 'start', '-i', compilation_container], stdout=logfile, cwd=testcase_folder) except Exception as e: print('An error occurred when compiling with docker.') grade_items_logging.log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) finally: if compilation_container != None: subprocess.call(['docker', 'rm', '-f', compilation_container]) print("cleaned up compilation container.") else: compile_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(testcase_folder,"my_compile.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string, '--testcase', str(testcase_num)], stdout=logfile, cwd=testcase_folder) # remove the compilation program untrusted_grant_rwx_access(which_untrusted, testcase_folder) os.remove(os.path.join(testcase_folder,"my_compile.out")) if compile_success == 0: print (which_machine,which_untrusted,"COMPILATION OK") else: print (which_machine,which_untrusted,"COMPILATION FAILURE") grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="COMPILATION FAILURE") add_permissions_recursive(tmp_compilation, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # return to the main tmp directory os.chdir(tmp) # -------------------------------------------------------------------- # make the runner directory with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nRUNNER STARTS", file=f) tmp_work = os.path.join(tmp,"TMP_WORK") tmp_work_test_input = os.path.join(tmp_work, "test_input") tmp_work_submission = os.path.join(tmp_work, "submitted_files") tmp_work_compiled = os.path.join(tmp_work, "compiled_files") tmp_work_checkout = os.path.join(tmp_work, "checkout") os.mkdir(tmp_work) os.mkdir(tmp_work_test_input) os.mkdir(tmp_work_submission) os.mkdir(tmp_work_compiled) os.mkdir(tmp_work_checkout) os.chdir(tmp_work) # move all executable files from the compilation directory to the main tmp directory # Note: Must preserve the directory structure of compiled files (esp for Java) patterns_submission_to_runner = complete_config_obj["autograding"]["submission_to_runner"] pattern_copy("submission_to_runner",patterns_submission_to_runner,submission_path,tmp_work_submission,tmp_logs) if is_vcs: pattern_copy("checkout_to_runner",patterns_submission_to_runner,checkout_subdir_path,tmp_work_checkout,tmp_logs) # move the compiled files into the tmp_work_compiled directory for path, name in executable_path_list: if not os.path.isfile(path): continue target_path = os.path.join(tmp_work_compiled, name) if not os.path.exists(target_path): os.makedirs(os.path.dirname(target_path), exist_ok=True) shutil.copy(path, target_path) print('copied over {0}'.format(target_path)) with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print('grade_item: copied over {0}'.format(target_path), file=f) patterns_compilation_to_runner = complete_config_obj["autograding"]["compilation_to_runner"] #copy into the actual tmp_work directory for archiving/validating pattern_copy("compilation_to_runner",patterns_compilation_to_runner,tmp_compilation,tmp_work,tmp_logs) #copy into tmp_work_compiled, which is provided to each testcase # TODO change this as our methodology for declaring testcase dependencies becomes more robust pattern_copy("compilation_to_runner",patterns_compilation_to_runner,tmp_compilation,tmp_work_compiled,tmp_logs) # copy input files to tmp_work directory copy_contents_into(job_id,test_input_path,tmp_work_test_input,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy runner.out to the current directory shutil.copy (os.path.join(bin_path,"run.out"),os.path.join(tmp_work,"my_runner.out")) #set the appropriate permissions for the newly created directories #TODO replaces commented out code below add_permissions(os.path.join(tmp_work,"my_runner.out"), stat.S_IXUSR | stat.S_IXGRP |stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_submission, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_compiled, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_checkout, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #TODO this is how permissions used to be set. It was removed because of the way it interacts with the sticky bit. ## give the untrusted user read/write/execute permissions on the tmp directory & files # os.system('ls -al {0}'.format(tmp_work)) # add_permissions_recursive(tmp_work, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) ################################################################################################## #call grade_item_main_runner.py runner_success = grade_item_main_runner.executeTestcases(complete_config_obj, tmp_logs, tmp_work, queue_obj, submission_string, item_name, USE_DOCKER, None, which_untrusted, job_id, grading_began) ################################################################################################## if runner_success == 0: print (which_machine,which_untrusted, "RUNNER OK") else: print (which_machine,which_untrusted, "RUNNER FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="RUNNER FAILURE") add_permissions_recursive(tmp_work, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions_recursive(tmp_compilation, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # -------------------------------------------------------------------- # RUN VALIDATOR with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nVALIDATION STARTS", file=f) # copy results files from compilation... patterns_submission_to_validation = complete_config_obj["autograding"]["submission_to_validation"] pattern_copy("submission_to_validation",patterns_submission_to_validation,submission_path,tmp_work,tmp_logs) if is_vcs: pattern_copy("checkout_to_validation",patterns_submission_to_validation,checkout_subdir_path,tmp_work,tmp_logs) patterns_compilation_to_validation = complete_config_obj["autograding"]["compilation_to_validation"] pattern_copy("compilation_to_validation",patterns_compilation_to_validation,tmp_compilation,tmp_work,tmp_logs) # remove the compilation directory shutil.rmtree(tmp_compilation) # copy output files to tmp_work directory copy_contents_into(job_id,test_output_path,tmp_work,tmp_logs) # copy any instructor custom validation code into the tmp work directory copy_contents_into(job_id,custom_validation_code_path,tmp_work,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy validator.out to the current directory shutil.copy (os.path.join(bin_path,"validate.out"),os.path.join(tmp_work,"my_validator.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(os.path.join(tmp_work,"my_validator.out"), stat.S_IXUSR | stat.S_IXGRP |stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #todo remove prints. print("VALIDATING") # validator the validator.out as the untrusted user with open(os.path.join(tmp_logs,"validator_log.txt"), 'w') as logfile: if USE_DOCKER: # WIP: This option file facilitated testing... #USE_DOCKER = os.path.isfile("/tmp/use_docker") #use_docker_string="grading begins, using DOCKER" if USE_DOCKER else "grading begins (not using docker)" #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message=use_docker_string) container = subprocess.check_output(['docker', 'run', '-t', '-d', '-v', tmp + ':' + tmp, 'ubuntu:custom']).decode('utf8').strip() dockerlaunch_done=dateutils.get_current_time() dockerlaunch_time = (dockerlaunch_done-grading_began).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,"dcct:",dockerlaunch_time,"docker container created") validator_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_validator.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string], stdout=logfile) else: validator_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"sbin","untrusted_execute"), which_untrusted, os.path.join(tmp_work,"my_validator.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string], stdout=logfile) if validator_success == 0: print (which_machine,which_untrusted,"VALIDATOR OK") else: print (which_machine,which_untrusted,"VALIDATOR FAILURE") grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="VALIDATION FAILURE") untrusted_grant_rwx_access(which_untrusted,tmp_work) # grab the result of autograding grade_result = "" try: with open(os.path.join(tmp_work,"grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open ",os.path.join(tmp_work,"grade.txt")) grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="ERROR: grade.txt does not exist") grade_items_logging.log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) # -------------------------------------------------------------------- # MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE tmp_results = os.path.join(tmp,"TMP_RESULTS") with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nARCHIVING STARTS", file=f) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) os.makedirs(os.path.join(tmp_results,"details")) # remove the test_input directory, so we don't archive it! shutil.rmtree(os.path.join(tmp_work,"test_input")) # loop over the test case directories, and remove any files that are also in the test_input folder for testcase_num in range(1, len(my_testcases)+1): testcase_folder = os.path.join(tmp_work, "test{:02}".format(testcase_num)) remove_test_input_files(os.path.join(tmp_logs,"overall.txt"),test_input_path,testcase_folder) patterns_work_to_details = complete_config_obj["autograding"]["work_to_details"] pattern_copy("work_to_details",patterns_work_to_details,tmp_work,os.path.join(tmp_results,"details"),tmp_logs) if ("work_to_public" in complete_config_obj["autograding"] and len(complete_config_obj["autograding"]["work_to_public"]) > 0): # create the directory os.makedirs(os.path.join(tmp_results,"results_public")) # copy the files patterns_work_to_public = complete_config_obj["autograding"]["work_to_public"] pattern_copy("work_to_public",patterns_work_to_public,tmp_work,os.path.join(tmp_results,"results_public"),tmp_logs) history_file_tmp = os.path.join(tmp_submission,"history.json") history_file = os.path.join(tmp_results,"history.json") if os.path.isfile(history_file_tmp): shutil.move(history_file_tmp,history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file,int(DAEMON_UID),ta_group_id) add_permissions(history_file,stat.S_IRGRP) grading_finished = dateutils.get_current_time() try: shutil.copy(os.path.join(tmp_work,"grade.txt"),tmp_results) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not copy ",os.path.join(tmp_work,"grade.txt")) grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="ERROR: grade.txt does not exist") grade_items_logging.log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) # ------------------------------------------------------------- # create/append to the results history # grab the submission time with open (os.path.join(submission_path,".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) gradeable_deadline_datetime = dateutils.read_submitty_date(gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date(gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int((submission_datetime-gradeable_deadline_datetime).total_seconds()) # note: negative = not late grading_finished_longstring = dateutils.write_submitty_date(grading_finished) gradingtime = (grading_finished-grading_began).total_seconds() with open(os.path.join(tmp_submission,"queue_file.json"), 'r') as infile: queue_obj = json.load(infile) queue_obj["gradingtime"]=gradingtime queue_obj["grade_result"]=grade_result queue_obj["which_untrusted"]=which_untrusted with open(os.path.join(tmp_results,"queue_file.json"),'w') as outfile: json.dump(queue_obj,outfile,sort_keys=True,indent=4,separators=(',', ': ')) try: shutil.move(os.path.join(tmp_work, "results.json"), os.path.join(tmp_results, "results.json")) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open/write ",os.path.join(tmp_work,"results.json")) grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,message="ERROR: results.json read/write error") grade_items_logging.log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) write_grade_history.just_write_grade_history(history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_time_longstring, is_batch_job_string, grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result, revision) os.chdir(SUBMITTY_DATA_DIR) if USE_DOCKER: with open(os.path.join(tmp_logs,"overall_log.txt"), 'w') as logfile: chmod_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container, 'chmod', '-R', 'ugo+rwx', '.'], stdout=logfile) with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: f.write("FINISHED GRADING!\n") # save the logs! shutil.copytree(tmp_logs,os.path.join(tmp_results,"logs")) # zip up results folder filehandle, my_results_zip_file=tempfile.mkstemp() zip_my_directory(tmp_results,my_results_zip_file) os.close(filehandle) shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp_results) shutil.rmtree(tmp_work) shutil.rmtree(tmp) # WIP: extra logging for testing #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message="done grading") # -------------------------------------------------------------------- # CLEAN UP DOCKER if USE_DOCKER: subprocess.call(['docker', 'rm', '-f', container]) dockerdestroy_done=dateutils.get_current_time() dockerdestroy_time = (dockerdestroy_done-grading_finished).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,"ddt:",dockerdestroy_time,"docker container destroyed") grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,item_name,"grade:",gradingtime,grade_result) return my_results_zip_file
def prepare_autograding_and_submission_zip(which_machine, which_untrusted, next_directory, next_to_grade): os.chdir(SUBMITTY_DATA_DIR) # generate a random id to be used to track this job in the autograding logs job_id = ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(6)) # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = load_queue_file_obj(job_id, next_directory, next_to_grade) partial_path = os.path.join(obj["gradeable"], obj["who"], str(obj["version"])) item_name = os.path.join(obj["semester"], obj["course"], "submissions", partial_path) submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name) if not os.path.isdir(submission_path): grade_items_logging.log_message( job_id, message="ERROR: the submission directory does not exist " + submission_path) raise RuntimeError("ERROR: the submission directory does not exist", submission_path) print(which_machine, which_untrusted, "prepare zip", submission_path) is_vcs, vcs_type, vcs_base_url, vcs_subdirectory = get_vcs_info( SUBMITTY_DATA_DIR, obj["semester"], obj["course"], obj["gradeable"], obj["who"], obj["team"]) is_batch_job = "regrade" in obj and obj["regrade"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory, next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = (grading_began - queue_time).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, "zip", item_name, "wait:", waittime, "") # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "provided_code", obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_input", obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_output", obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "custom_validation_code", obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "bin", obj["gradeable"]) form_json_config = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "form", "form_" + obj["gradeable"] + ".json") complete_config = os.path.join( SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "complete_config", "complete_config_" + obj["gradeable"] + ".json") if not os.path.exists(form_json_config): grade_items_logging.log_message( job_id, message="ERROR: the form json file does not exist " + form_json_config) raise RuntimeError("ERROR: the form json file does not exist ", form_json_config) if not os.path.exists(complete_config): grade_items_logging.log_message( job_id, message="ERROR: the complete config file does not exist " + complete_config) raise RuntimeError("ERROR: the complete config file does not exist ", complete_config) # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = tempfile.mkdtemp() tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") os.mkdir(tmp_autograding) tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") os.mkdir(tmp_submission) copytree_if_exists(provided_code_path, os.path.join(tmp_autograding, "provided_code")) copytree_if_exists(test_input_path, os.path.join(tmp_autograding, "test_input")) copytree_if_exists(test_output_path, os.path.join(tmp_autograding, "test_output")) copytree_if_exists(custom_validation_code_path, os.path.join(tmp_autograding, "custom_validation_code")) copytree_if_exists(bin_path, os.path.join(tmp_autograding, "bin")) shutil.copy(form_json_config, os.path.join(tmp_autograding, "form.json")) shutil.copy(complete_config, os.path.join(tmp_autograding, "complete_config.json")) checkout_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "checkout", partial_path) results_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "results", partial_path) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path, "history.json") history_file_tmp = "" if os.path.isfile(history_file): shutil.copy(history_file, os.path.join(tmp_submission, "history.json")) # get info from the gradeable config file with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) queue_file = os.path.join(next_directory, next_to_grade) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") os.makedirs(tmp_logs) # 'touch' a file in the logs folder open(os.path.join(tmp_logs, "overall.txt"), 'a') # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) # -------------------------------------------------------------------- # CHECKOUT THE STUDENT's REPO if is_vcs: # cleanup the previous checkout (if it exists) shutil.rmtree(checkout_path, ignore_errors=True) os.makedirs(checkout_path, exist_ok=True) try: # If we are public or private github, we will have an empty vcs_subdirectory if vcs_subdirectory == '': with open(os.path.join( submission_path, ".submit.VCS_CHECKOUT")) as submission_vcs_file: VCS_JSON = json.load(submission_vcs_file) git_user_id = VCS_JSON["git_user_id"] git_repo_id = VCS_JSON["git_repo_id"] if not valid_github_user_id(git_user_id): raise Exception( "Invalid GitHub user/organization name: '" + git_user_id + "'") if not valid_github_repo_id(git_repo_id): raise Exception("Invalid GitHub repository name: '" + git_repo_id + "'") # construct path for GitHub vcs_path = "https://www.github.com/" + git_user_id + "/" + git_repo_id # is vcs_subdirectory standalone or should it be combined with base_url? elif vcs_subdirectory[0] == '/' or '://' in vcs_subdirectory: vcs_path = vcs_subdirectory else: if '://' in vcs_base_url: vcs_path = urllib.parse.urljoin(vcs_base_url, vcs_subdirectory) else: vcs_path = os.path.join(vcs_base_url, vcs_subdirectory) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nVCS CHECKOUT", file=f) print('vcs_base_url', vcs_base_url, file=f) print('vcs_subdirectory', vcs_subdirectory, file=f) print('vcs_path', vcs_path, file=f) print(['/usr/bin/git', 'clone', vcs_path, checkout_path], file=f) # git clone may fail -- because repository does not exist, # or because we don't have appropriate access credentials try: subprocess.check_call( ['/usr/bin/git', 'clone', vcs_path, checkout_path]) os.chdir(checkout_path) # determine which version we need to checkout # if the repo is empty or the master branch does not exist, this command will fail try: what_version = subprocess.check_output([ 'git', 'rev-list', '-n', '1', '--before="' + submission_string + '"', 'master' ]) what_version = str(what_version.decode('utf-8')).rstrip() if what_version == "": # oops, pressed the grade button before a valid commit shutil.rmtree(checkout_path, ignore_errors=True) else: # and check out the right version subprocess.call( ['git', 'checkout', '-b', 'grade', what_version]) os.chdir(tmp) subprocess.call(['ls', '-lR', checkout_path], stdout=open(tmp_logs + "/overall.txt", 'a')) obj['revision'] = what_version # exception on git rev-list except subprocess.CalledProcessError as error: grade_items_logging.log_message( job_id, message= "ERROR: failed to determine version on master branch " + str(error)) os.chdir(checkout_path) with open( os.path.join( checkout_path, "failed_to_determine_version_on_master_branch.txt" ), 'w') as f: print(str(error), file=f) print("\n", file=f) print( "Check to be sure the repository is not empty.\n", file=f) print( "Check to be sure the repository has a master branch.\n", file=f) print( "And check to be sure the timestamps on the master branch are reasonable.\n", file=f) # exception on git clone except subprocess.CalledProcessError as error: grade_items_logging.log_message( job_id, message="ERROR: failed to clone repository " + str(error)) os.chdir(checkout_path) with open( os.path.join(checkout_path, "failed_to_clone_repository.txt"), 'w') as f: print(str(error), file=f) print("\n", file=f) print("Check to be sure the repository exists.\n", file=f) print( "And check to be sure the submitty_daemon user has appropriate access credentials.\n", file=f) # exception in constructing full git repository url/path except Exception as error: grade_items_logging.log_message( job_id, message="ERROR: failed to construct valid repository url/path" + str(error)) os.chdir(checkout_path) with open( os.path.join( checkout_path, "failed_to_construct_valid_repository_url.txt"), 'w') as f: print(str(error), file=f) print("\n", file=f) print("Check to be sure the repository exists.\n", file=f) print( "And check to be sure the submitty_daemon user has appropriate access credentials.\n", file=f) copytree_if_exists(submission_path, os.path.join(tmp_submission, "submission")) copytree_if_exists(checkout_path, os.path.join(tmp_submission, "checkout")) obj["queue_time"] = queue_time_longstring obj["regrade"] = is_batch_job obj["waittime"] = waittime obj["job_id"] = job_id with open(os.path.join(tmp_submission, "queue_file.json"), 'w') as outfile: json.dump(obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) grading_began_longstring = dateutils.write_submitty_date(grading_began) with open(os.path.join(tmp_submission, ".grading_began"), 'w') as f: print(grading_began_longstring, file=f) # zip up autograding & submission folders filehandle1, my_autograding_zip_file = tempfile.mkstemp() filehandle2, my_submission_zip_file = tempfile.mkstemp() grade_item.zip_my_directory(tmp_autograding, my_autograding_zip_file) grade_item.zip_my_directory(tmp_submission, my_submission_zip_file) os.close(filehandle1) os.close(filehandle2) # cleanup shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp) #grade_items_logging.log_message(job_id,is_batch_job,"done zip",item_name) return (my_autograding_zip_file, my_submission_zip_file)
def grade_from_zip(my_autograding_zip_file, my_submission_zip_file, which_untrusted): os.chdir(SUBMITTY_DATA_DIR) tmp = os.path.join("/var/local/submitty/autograding_tmp/", which_untrusted, "tmp") if os.path.exists(tmp): untrusted_grant_rwx_access(which_untrusted, tmp) add_permissions_recursive( tmp, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # Remove any and all containers left over from past runs. old_containers = subprocess.check_output( ['docker', 'ps', '-aq', '-f', 'name={0}'.format(which_untrusted)]).split() for old_container in old_containers: subprocess.call(['docker', 'rm', '-f', old_container.decode('utf8')]) # clean up old usage of this directory shutil.rmtree(tmp, ignore_errors=True) os.mkdir(tmp) which_machine = socket.gethostname() # unzip autograding and submission folders tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") try: unzip_this_file(my_autograding_zip_file, tmp_autograding) unzip_this_file(my_submission_zip_file, tmp_submission) except: raise os.remove(my_autograding_zip_file) os.remove(my_submission_zip_file) tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") queue_file = os.path.join(tmp_submission, "queue_file.json") with open(queue_file, 'r') as infile: queue_obj = json.load(infile) queue_time_longstring = queue_obj["queue_time"] waittime = queue_obj["waittime"] is_batch_job = queue_obj["regrade"] job_id = queue_obj["job_id"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" revision = queue_obj.get("revision", None) partial_path = os.path.join(queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"], queue_obj["course"], "submissions", partial_path) grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "wait:", waittime, "") with open(os.path.join(tmp_submission, ".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) submission_path = os.path.join(tmp_submission, "submission") checkout_path = os.path.join(tmp_submission, "checkout") provided_code_path = os.path.join(tmp_autograding, "provided_code") test_input_path = os.path.join(tmp_autograding, "test_input") test_output_path = os.path.join(tmp_autograding, "test_output") custom_validation_code_path = os.path.join(tmp_autograding, "custom_validation_code") bin_path = os.path.join(tmp_autograding, "bin") form_json_config = os.path.join(tmp_autograding, "form.json") complete_config = os.path.join(tmp_autograding, "complete_config.json") with open(form_json_config, 'r') as infile: gradeable_config_obj = json.load(infile) gradeable_deadline_string = gradeable_config_obj["date_due"] with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) is_vcs = gradeable_config_obj["upload_type"] == "repository" checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) if complete_config_obj.get('one_part_only', False): allow_only_one_part(submission_path, os.path.join(tmp_logs, "overall.txt")) if is_vcs: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print( "WARNING: ONE_PART_ONLY OPTION DOES NOT MAKE SENSE WITH VCS SUBMISSION", file=f) # -------------------------------------------------------------------- # START DOCKER # NOTE: DOCKER SUPPORT PRELIMINARY -- NEEDS MORE SECURITY BEFORE DEPLOYED ON LIVE SERVER complete_config = os.path.join(tmp_autograding, "complete_config.json") with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) # Save ourselves if autograding_method is None. autograding_method = complete_config_obj.get("autograding_method", "") USE_DOCKER = True if autograding_method == "docker" else False # -------------------------------------------------------------------- # COMPILE THE SUBMITTED CODE with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nCOMPILATION STARTS", file=f) # copy submitted files to the tmp compilation directory tmp_compilation = os.path.join(tmp, "TMP_COMPILATION") os.mkdir(tmp_compilation) os.chdir(tmp_compilation) submission_path = os.path.join(tmp_submission, "submission") checkout_path = os.path.join(tmp_submission, "checkout") provided_code_path = os.path.join(tmp_autograding, "provided_code") test_input_path = os.path.join(tmp_autograding, "test_input") test_output_path = os.path.join(tmp_autograding, "test_output") custom_validation_code_path = os.path.join(tmp_autograding, "custom_validation_code") bin_path = os.path.join(tmp_autograding, "bin") form_json_config = os.path.join(tmp_autograding, "form.json") with open(form_json_config, 'r') as infile: gradeable_config_obj = json.load(infile) gradeable_deadline_string = gradeable_config_obj["date_due"] patterns_submission_to_compilation = complete_config_obj["autograding"][ "submission_to_compilation"] add_permissions(tmp_logs, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) if USE_DOCKER: print("!!!!!!!!!!!!!!!!!!USING DOCKER!!!!!!!!!!!!!!!!!!!!!!!!") with open(complete_config, 'r') as infile: config = json.load(infile) my_testcases = config['testcases'] # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() with open(os.path.join(tmp_logs, "compilation_log.txt"), 'w') as logfile: # we start counting from one. executable_path_list = list() for testcase_num in range(1, len(my_testcases) + 1): testcase_folder = os.path.join(tmp_compilation, "test{:02}".format(testcase_num)) if 'type' in my_testcases[testcase_num - 1]: if my_testcases[testcase_num - 1]['type'] != 'FileCheck' and my_testcases[ testcase_num - 1]['type'] != 'Compilation': continue if my_testcases[testcase_num - 1]['type'] == 'Compilation': if 'executable_name' in my_testcases[testcase_num - 1]: provided_executable_list = my_testcases[ testcase_num - 1]['executable_name'] if not isinstance(provided_executable_list, (list, )): provided_executable_list = list( [provided_executable_list]) for executable_name in provided_executable_list: if executable_name.strip() == '': continue executable_path = os.path.join( testcase_folder, executable_name) executable_path_list.append( (executable_path, executable_name)) else: continue os.makedirs(testcase_folder) pattern_copy("submission_to_compilation", patterns_submission_to_compilation, submission_path, testcase_folder, tmp_logs) if is_vcs: pattern_copy("checkout_to_compilation", patterns_submission_to_compilation, checkout_subdir_path, testcase_folder, tmp_logs) # copy any instructor provided code files to tmp compilation directory copy_contents_into(job_id, provided_code_path, testcase_folder, tmp_logs) # copy compile.out to the current directory shutil.copy(os.path.join(bin_path, "compile.out"), os.path.join(testcase_folder, "my_compile.out")) add_permissions( os.path.join(testcase_folder, "my_compile.out"), stat.S_IXUSR | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #untrusted_grant_rwx_access(which_untrusted, tmp_compilation) untrusted_grant_rwx_access(which_untrusted, testcase_folder) add_permissions_recursive( testcase_folder, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) if USE_DOCKER: try: #There can be only one container for a compilation step, so grab its container image #TODO: set default in load_config_json.cpp if my_testcases[testcase_num - 1]['type'] == 'FileCheck': print( "performing filecheck in default ubuntu:custom container" ) container_image = "ubuntu:custom" else: container_image = my_testcases[ testcase_num - 1]["containers"][0]["container_image"] print( 'creating a compilation container with image {0}'. format(container_image)) untrusted_uid = str(getpwnam(which_untrusted).pw_uid) compilation_container = None compilation_container = subprocess.check_output([ 'docker', 'create', '-i', '-u', untrusted_uid, '--network', 'none', '-v', testcase_folder + ':' + testcase_folder, '-w', testcase_folder, container_image, #The command to be run. os.path.join(testcase_folder, 'my_compile.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string, '--testcase', str(testcase_num) ]).decode('utf8').strip() print("starting container") compile_success = subprocess.call( ['docker', 'start', '-i', compilation_container], stdout=logfile, cwd=testcase_folder) except Exception as e: print('An error occurred when compiling with docker.') grade_items_logging.log_stack_trace( job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) finally: if compilation_container != None: subprocess.call( ['docker', 'rm', '-f', compilation_container]) print("cleaned up compilation container.") else: compile_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(testcase_folder, "my_compile.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string, '--testcase', str(testcase_num) ], stdout=logfile, cwd=testcase_folder) # remove the compilation program untrusted_grant_rwx_access(which_untrusted, testcase_folder) os.remove(os.path.join(testcase_folder, "my_compile.out")) if compile_success == 0: print(which_machine, which_untrusted, "COMPILATION OK") else: print(which_machine, which_untrusted, "COMPILATION FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="COMPILATION FAILURE") add_permissions_recursive( tmp_compilation, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # return to the main tmp directory os.chdir(tmp) # -------------------------------------------------------------------- # make the runner directory with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nRUNNER STARTS", file=f) tmp_work = os.path.join(tmp, "TMP_WORK") tmp_work_test_input = os.path.join(tmp_work, "test_input") tmp_work_submission = os.path.join(tmp_work, "submitted_files") tmp_work_compiled = os.path.join(tmp_work, "compiled_files") tmp_work_checkout = os.path.join(tmp_work, "checkout") os.mkdir(tmp_work) os.mkdir(tmp_work_test_input) os.mkdir(tmp_work_submission) os.mkdir(tmp_work_compiled) os.mkdir(tmp_work_checkout) os.chdir(tmp_work) # move all executable files from the compilation directory to the main tmp directory # Note: Must preserve the directory structure of compiled files (esp for Java) patterns_submission_to_runner = complete_config_obj["autograding"][ "submission_to_runner"] pattern_copy("submission_to_runner", patterns_submission_to_runner, submission_path, tmp_work_submission, tmp_logs) if is_vcs: pattern_copy("checkout_to_runner", patterns_submission_to_runner, checkout_subdir_path, tmp_work_checkout, tmp_logs) # move the compiled files into the tmp_work_compiled directory for path, name in executable_path_list: if not os.path.isfile(path): continue target_path = os.path.join(tmp_work_compiled, name) if not os.path.exists(target_path): os.makedirs(os.path.dirname(target_path), exist_ok=True) shutil.copy(path, target_path) print('copied over {0}'.format(target_path)) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print('grade_item: copied over {0}'.format(target_path), file=f) patterns_compilation_to_runner = complete_config_obj["autograding"][ "compilation_to_runner"] #copy into the actual tmp_work directory for archiving/validating pattern_copy("compilation_to_runner", patterns_compilation_to_runner, tmp_compilation, tmp_work, tmp_logs) #copy into tmp_work_compiled, which is provided to each testcase # TODO change this as our methodology for declaring testcase dependencies becomes more robust pattern_copy("compilation_to_runner", patterns_compilation_to_runner, tmp_compilation, tmp_work_compiled, tmp_logs) # copy input files to tmp_work directory copy_contents_into(job_id, test_input_path, tmp_work_test_input, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy runner.out to the current directory shutil.copy(os.path.join(bin_path, "run.out"), os.path.join(tmp_work, "my_runner.out")) #set the appropriate permissions for the newly created directories #TODO replaces commented out code below add_permissions( os.path.join(tmp_work, "my_runner.out"), stat.S_IXUSR | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_submission, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_compiled, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(tmp_work_checkout, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #TODO this is how permissions used to be set. It was removed because of the way it interacts with the sticky bit. ## give the untrusted user read/write/execute permissions on the tmp directory & files # os.system('ls -al {0}'.format(tmp_work)) # add_permissions_recursive(tmp_work, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, # stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) ################################################################################################## #call grade_item_main_runner.py runner_success = grade_item_main_runner.executeTestcases( complete_config_obj, tmp_logs, tmp_work, queue_obj, submission_string, item_name, USE_DOCKER, None, which_untrusted, job_id, grading_began) ################################################################################################## if runner_success == 0: print(which_machine, which_untrusted, "RUNNER OK") else: print(which_machine, which_untrusted, "RUNNER FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="RUNNER FAILURE") add_permissions_recursive( tmp_work, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions_recursive( tmp_compilation, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # -------------------------------------------------------------------- # RUN VALIDATOR with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nVALIDATION STARTS", file=f) # copy results files from compilation... patterns_submission_to_validation = complete_config_obj["autograding"][ "submission_to_validation"] pattern_copy("submission_to_validation", patterns_submission_to_validation, submission_path, tmp_work, tmp_logs) if is_vcs: pattern_copy("checkout_to_validation", patterns_submission_to_validation, checkout_subdir_path, tmp_work, tmp_logs) patterns_compilation_to_validation = complete_config_obj["autograding"][ "compilation_to_validation"] pattern_copy("compilation_to_validation", patterns_compilation_to_validation, tmp_compilation, tmp_work, tmp_logs) # remove the compilation directory shutil.rmtree(tmp_compilation) # copy output files to tmp_work directory copy_contents_into(job_id, test_output_path, tmp_work, tmp_logs) # copy any instructor custom validation code into the tmp work directory copy_contents_into(job_id, custom_validation_code_path, tmp_work, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy validator.out to the current directory shutil.copy(os.path.join(bin_path, "validate.out"), os.path.join(tmp_work, "my_validator.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions( os.path.join(tmp_work, "my_validator.out"), stat.S_IXUSR | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) #todo remove prints. print("VALIDATING") # validator the validator.out as the untrusted user with open(os.path.join(tmp_logs, "validator_log.txt"), 'w') as logfile: if USE_DOCKER: # WIP: This option file facilitated testing... #USE_DOCKER = os.path.isfile("/tmp/use_docker") #use_docker_string="grading begins, using DOCKER" if USE_DOCKER else "grading begins (not using docker)" #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message=use_docker_string) container = subprocess.check_output([ 'docker', 'run', '-t', '-d', '-v', tmp + ':' + tmp, 'ubuntu:custom' ]).decode('utf8').strip() dockerlaunch_done = dateutils.get_current_time() dockerlaunch_time = (dockerlaunch_done - grading_began).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "dcct:", dockerlaunch_time, "docker container created") validator_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_validator.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string ], stdout=logfile) else: validator_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(tmp_work, "my_validator.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string ], stdout=logfile) if validator_success == 0: print(which_machine, which_untrusted, "VALIDATOR OK") else: print(which_machine, which_untrusted, "VALIDATOR FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="VALIDATION FAILURE") untrusted_grant_rwx_access(which_untrusted, tmp_work) # grab the result of autograding grade_result = "" try: with open(os.path.join(tmp_work, "grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line except: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("\n\nERROR: Grading incomplete -- Could not open ", os.path.join(tmp_work, "grade.txt")) grade_items_logging.log_message( job_id, is_batch_job, which_untrusted, item_name, message="ERROR: grade.txt does not exist") grade_items_logging.log_stack_trace(job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) # -------------------------------------------------------------------- # MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE tmp_results = os.path.join(tmp, "TMP_RESULTS") with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nARCHIVING STARTS", file=f) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) os.makedirs(os.path.join(tmp_results, "details")) # remove the test_input directory, so we don't archive it! shutil.rmtree(os.path.join(tmp_work, "test_input")) # loop over the test case directories, and remove any files that are also in the test_input folder for testcase_num in range(1, len(my_testcases) + 1): testcase_folder = os.path.join(tmp_work, "test{:02}".format(testcase_num)) remove_test_input_files(os.path.join(tmp_logs, "overall.txt"), test_input_path, testcase_folder) patterns_work_to_details = complete_config_obj["autograding"][ "work_to_details"] pattern_copy("work_to_details", patterns_work_to_details, tmp_work, os.path.join(tmp_results, "details"), tmp_logs) if ("work_to_public" in complete_config_obj["autograding"] and len(complete_config_obj["autograding"]["work_to_public"]) > 0): # create the directory os.makedirs(os.path.join(tmp_results, "results_public")) # copy the files patterns_work_to_public = complete_config_obj["autograding"][ "work_to_public"] pattern_copy("work_to_public", patterns_work_to_public, tmp_work, os.path.join(tmp_results, "results_public"), tmp_logs) history_file_tmp = os.path.join(tmp_submission, "history.json") history_file = os.path.join(tmp_results, "history.json") if os.path.isfile(history_file_tmp): shutil.move(history_file_tmp, history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file, int(DAEMON_UID), ta_group_id) add_permissions(history_file, stat.S_IRGRP) grading_finished = dateutils.get_current_time() try: shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results) except: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("\n\nERROR: Grading incomplete -- Could not copy ", os.path.join(tmp_work, "grade.txt")) grade_items_logging.log_message( job_id, is_batch_job, which_untrusted, item_name, message="ERROR: grade.txt does not exist") grade_items_logging.log_stack_trace(job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) # ------------------------------------------------------------- # create/append to the results history # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) gradeable_deadline_datetime = dateutils.read_submitty_date( gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date( gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int( (submission_datetime - gradeable_deadline_datetime).total_seconds()) # note: negative = not late grading_finished_longstring = dateutils.write_submitty_date( grading_finished) gradingtime = (grading_finished - grading_began).total_seconds() with open(os.path.join(tmp_submission, "queue_file.json"), 'r') as infile: queue_obj = json.load(infile) queue_obj["gradingtime"] = gradingtime queue_obj["grade_result"] = grade_result queue_obj["which_untrusted"] = which_untrusted with open(os.path.join(tmp_results, "queue_file.json"), 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) try: shutil.move(os.path.join(tmp_work, "results.json"), os.path.join(tmp_results, "results.json")) except: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("\n\nERROR: Grading incomplete -- Could not open/write ", os.path.join(tmp_work, "results.json")) grade_items_logging.log_message( job_id, is_batch_job, which_untrusted, item_name, message="ERROR: results.json read/write error") grade_items_logging.log_stack_trace(job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) write_grade_history.just_write_grade_history( history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_time_longstring, is_batch_job_string, grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result, revision) os.chdir(SUBMITTY_DATA_DIR) if USE_DOCKER: with open(os.path.join(tmp_logs, "overall_log.txt"), 'w') as logfile: chmod_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, 'chmod', '-R', 'ugo+rwx', '.' ], stdout=logfile) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: f.write("FINISHED GRADING!\n") # save the logs! shutil.copytree(tmp_logs, os.path.join(tmp_results, "logs")) # zip up results folder filehandle, my_results_zip_file = tempfile.mkstemp() zip_my_directory(tmp_results, my_results_zip_file) os.close(filehandle) shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp_results) shutil.rmtree(tmp_work) shutil.rmtree(tmp) # WIP: extra logging for testing #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message="done grading") # -------------------------------------------------------------------- # CLEAN UP DOCKER if USE_DOCKER: subprocess.call(['docker', 'rm', '-f', container]) dockerdestroy_done = dateutils.get_current_time() dockerdestroy_time = (dockerdestroy_done - grading_finished).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "ddt:", dockerdestroy_time, "docker container destroyed") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "grade:", gradingtime, grade_result) return my_results_zip_file
def prepare_autograding_and_submission_zip(next_directory, next_to_grade): os.chdir(SUBMITTY_DATA_DIR) # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = load_queue_file_obj(next_directory, next_to_grade) partial_path = os.path.join(obj["gradeable"], obj["who"], str(obj["version"])) item_name = os.path.join(obj["semester"], obj["course"], "submissions", partial_path) submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name) if not os.path.isdir(submission_path): grade_items_logging.log_message( message="ERROR: the submission directory does not exist" + submission_path) raise RuntimeError("ERROR: the submission directory does not exist", submission_path) print("pid", os.getpid(), "GRADE THIS", submission_path) is_vcs, vcs_type, vcs_base_url, vcs_subdirectory = get_vcs_info( SUBMITTY_DATA_DIR, obj["semester"], obj["course"], obj["gradeable"], obj["who"], obj["team"]) is_batch_job = next_directory == BATCH_QUEUE is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory, next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = (grading_began - queue_time).total_seconds() grade_items_logging.log_message(is_batch_job, "zip", item_name, "wait:", waittime, "") # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = tempfile.mkdtemp() tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") os.mkdir(tmp_autograding) tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") os.mkdir(tmp_submission) # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "provided_code", obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_input", obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_output", obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "custom_validation_code", obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "bin", obj["gradeable"]) form_json_config = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "form", "form_" + obj["gradeable"] + ".json") complete_config = os.path.join( SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "complete_config", "complete_config_" + obj["gradeable"] + ".json") copytree_if_exists(provided_code_path, os.path.join(tmp_autograding, "provided_code")) copytree_if_exists(test_input_path, os.path.join(tmp_autograding, "test_input")) copytree_if_exists(test_output_path, os.path.join(tmp_autograding, "test_output")) copytree_if_exists(custom_validation_code_path, os.path.join(tmp_autograding, "custom_validation_code")) copytree_if_exists(bin_path, os.path.join(tmp_autograding, "bin")) shutil.copy(form_json_config, os.path.join(tmp_autograding, "form.json")) shutil.copy(complete_config, os.path.join(tmp_autograding, "complete_config.json")) checkout_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "checkout", partial_path) results_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "results", partial_path) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path, "history.json") history_file_tmp = "" if os.path.isfile(history_file): filehandle, history_file_tmp = tempfile.mkstemp() shutil.copy(history_file, history_file_tmp) shutil.copy(history_file, os.path.join(tmp_submission, "history.json")) # get info from the gradeable config file with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) queue_file = os.path.join(next_directory, next_to_grade) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") os.makedirs(tmp_logs) # 'touch' a file in the logs folder open(os.path.join(tmp_logs, "overall.txt"), 'a') # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) # -------------------------------------------------------------------- # CHECKOUT THE STUDENT's REPO if is_vcs: # is vcs_subdirectory standalone or should it be combined with base_url? if vcs_subdirectory[0] == '/' or '://' in vcs_subdirectory: vcs_path = vcs_subdirectory else: if '://' in vcs_base_url: vcs_path = urllib.parse.urljoin(vcs_base_url, vcs_subdirectory) else: vcs_path = os.path.join(vcs_base_url, vcs_subdirectory) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nVCS CHECKOUT", file=f) print('vcs_base_url', vcs_base_url, file=f) print('vcs_subdirectory', vcs_subdirectory, file=f) print('vcs_path', vcs_path, file=f) print(['/usr/bin/git', 'clone', vcs_path, checkout_path], file=f) # cleanup the previous checkout (if it exists) shutil.rmtree(checkout_path, ignore_errors=True) os.makedirs(checkout_path, exist_ok=True) subprocess.call(['/usr/bin/git', 'clone', vcs_path, checkout_path]) os.chdir(checkout_path) # determine which version we need to checkout what_version = subprocess.check_output([ 'git', 'rev-list', '-n', '1', '--before="' + submission_string + '"', 'master' ]) what_version = str(what_version.decode('utf-8')).rstrip() if what_version == "": # oops, pressed the grade button before a valid commit shutil.rmtree(checkout_path, ignore_errors=True) else: # and check out the right version subprocess.call(['git', 'checkout', '-b', 'grade', what_version]) os.chdir(tmp) subprocess.call(['ls', '-lR', checkout_path], stdout=open(tmp_logs + "/overall.txt", 'a')) copytree_if_exists(submission_path, os.path.join(tmp_submission, "submission")) copytree_if_exists(checkout_path, os.path.join(tmp_submission, "checkout")) obj["queue_time"] = queue_time_longstring obj["is_batch_job"] = is_batch_job obj["waittime"] = waittime with open(os.path.join(tmp_submission, "queue_file.json"), 'w') as outfile: json.dump(obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) grading_began_longstring = dateutils.write_submitty_date(grading_began) with open(os.path.join(tmp_submission, ".grading_began"), 'w') as f: print(grading_began_longstring, file=f) # zip up autograding & submission folders my_autograding_zip_file = tempfile.mkstemp()[1] my_submission_zip_file = tempfile.mkstemp()[1] zip_my_directory(tmp_autograding, my_autograding_zip_file) zip_my_directory(tmp_submission, my_submission_zip_file) # cleanup shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp) return (my_autograding_zip_file, my_submission_zip_file)
def prepare_autograding_and_submission_zip(which_machine, which_untrusted, next_directory, next_to_grade): os.chdir(SUBMITTY_DATA_DIR) # generate a random id to be used to track this job in the autograding logs job_id = ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(6)) # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = load_queue_file_obj(job_id, next_directory, next_to_grade) partial_path = os.path.join(obj["gradeable"], obj["who"], str(obj["version"])) item_name = os.path.join(obj["semester"], obj["course"], "submissions", partial_path) submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name) if not os.path.isdir(submission_path): grade_items_logging.log_message( job_id, message="ERROR: the submission directory does not exist " + submission_path) raise RuntimeError("ERROR: the submission directory does not exist", submission_path) print(which_machine, which_untrusted, "prepare zip", submission_path) is_vcs, vcs_type, vcs_base_url, vcs_subdirectory = get_vcs_info( SUBMITTY_DATA_DIR, obj["semester"], obj["course"], obj["gradeable"], obj["who"], obj["team"]) is_batch_job = "regrade" in obj and obj["regrade"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory, next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = (grading_began - queue_time).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, "zip", item_name, "wait:", waittime, "") # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "provided_code", obj["gradeable"]) instructor_solution_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "instructor_solution", obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_input", obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_output", obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "custom_validation_code", obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "bin", obj["gradeable"]) form_json_config = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "form", "form_" + obj["gradeable"] + ".json") complete_config = os.path.join( SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "complete_config", "complete_config_" + obj["gradeable"] + ".json") if not os.path.exists(form_json_config): grade_items_logging.log_message( job_id, message="ERROR: the form json file does not exist " + form_json_config) raise RuntimeError("ERROR: the form json file does not exist ", form_json_config) if not os.path.exists(complete_config): grade_items_logging.log_message( job_id, message="ERROR: the complete config file does not exist " + complete_config) raise RuntimeError("ERROR: the complete config file does not exist ", complete_config) # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = tempfile.mkdtemp() tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") os.mkdir(tmp_autograding) tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") os.mkdir(tmp_submission) copytree_if_exists(provided_code_path, os.path.join(tmp_autograding, "provided_code")) copytree_if_exists(instructor_solution_path, os.path.join(tmp_autograding, "instructor_solution")) copytree_if_exists(test_input_path, os.path.join(tmp_autograding, "test_input")) copytree_if_exists(test_output_path, os.path.join(tmp_autograding, "test_output")) copytree_if_exists(custom_validation_code_path, os.path.join(tmp_autograding, "custom_validation_code")) copytree_if_exists(bin_path, os.path.join(tmp_autograding, "bin")) shutil.copy(form_json_config, os.path.join(tmp_autograding, "form.json")) shutil.copy(complete_config, os.path.join(tmp_autograding, "complete_config.json")) checkout_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "checkout", partial_path) results_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "results", partial_path) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path, "history.json") history_file_tmp = "" if os.path.isfile(history_file): shutil.copy(history_file, os.path.join(tmp_submission, "history.json")) # get info from the gradeable config file with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) queue_file = os.path.join(next_directory, next_to_grade) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") os.makedirs(tmp_logs) # 'touch' a file in the logs folder open(os.path.join(tmp_logs, "overall.txt"), 'a') # -------------------------------------------------------------------- # CONFIRM WE HAVE A CHECKOUT OF THE STUDENT'S REPO if is_vcs: # there should be a checkout log file in the results directory # move that file to the tmp logs directory.. vcs_checkout_logfile = os.path.join(results_path, "logs", "vcs_checkout.txt") if os.path.isfile(vcs_checkout_logfile): shutil.move(vcs_checkout_logfile, tmp_logs) else: grade_items_logging.log_message( JOB_ID, message=str(my_name) + " ERROR: missing vcs_checkout.txt logfile " + str(vcs_checkout_logfile)) copytree_if_exists(submission_path, os.path.join(tmp_submission, "submission")) copytree_if_exists(checkout_path, os.path.join(tmp_submission, "checkout")) obj["queue_time"] = queue_time_longstring obj["regrade"] = is_batch_job obj["waittime"] = waittime obj["job_id"] = job_id with open(os.path.join(tmp_submission, "queue_file.json"), 'w') as outfile: json.dump(obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) grading_began_longstring = dateutils.write_submitty_date(grading_began) with open(os.path.join(tmp_submission, ".grading_began"), 'w') as f: print(grading_began_longstring, file=f) # zip up autograding & submission folders filehandle1, my_autograding_zip_file = tempfile.mkstemp() filehandle2, my_submission_zip_file = tempfile.mkstemp() grade_item.zip_my_directory(tmp_autograding, my_autograding_zip_file) grade_item.zip_my_directory(tmp_submission, my_submission_zip_file) os.close(filehandle1) os.close(filehandle2) # cleanup shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp) #grade_items_logging.log_message(job_id,is_batch_job,"done zip",item_name) return (my_autograding_zip_file, my_submission_zip_file)
def archive_autograding_results(working_directory, job_id, which_untrusted, is_batch_job, complete_config_obj, gradeable_config_obj, queue_obj, log_path, stack_trace_log_path, is_test_environment): """ After grading is finished, archive the results. """ tmp_autograding = os.path.join(working_directory,"TMP_AUTOGRADING") tmp_submission = os.path.join(working_directory,"TMP_SUBMISSION") tmp_work = os.path.join(working_directory,"TMP_WORK") tmp_logs = os.path.join(working_directory,"TMP_SUBMISSION","tmp_logs") tmp_results = os.path.join(working_directory,"TMP_RESULTS") submission_path = os.path.join(tmp_submission, "submission") random_output_path = os.path.join(tmp_work, 'random_output') partial_path = os.path.join(queue_obj["gradeable"],queue_obj["who"],str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"],queue_obj["course"],"submissions",partial_path) results_public_dir = os.path.join(tmp_results,"results_public") results_details_dir = os.path.join(tmp_results, "details") patterns = complete_config_obj['autograding'] # Copy work to details pattern_copy("work_to_details", patterns['work_to_details'], tmp_work, results_details_dir, tmp_logs) # Copy work to public if 'work_to_public' in patterns: pattern_copy("work_to_public", patterns['work_to_public'], tmp_work, results_public_dir, tmp_logs) if os.path.exists(random_output_path): pattern_copy("work_to_random_output", [os.path.join(random_output_path, 'test*', '**', '*.txt'),], tmp_work, tmp_results, tmp_logs) # grab the submission time with open(os.path.join(tmp_submission, 'submission' ,".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() history_file_tmp = os.path.join(tmp_submission,"history.json") history_file = os.path.join(tmp_results,"history.json") if os.path.isfile(history_file_tmp) and not is_test_environment: from . import CONFIG_PATH with open(os.path.join(CONFIG_PATH, 'submitty_users.json')) as open_file: OPEN_JSON = json.load(open_file) DAEMON_UID = OPEN_JSON['daemon_uid'] shutil.move(history_file_tmp, history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file, int(DAEMON_UID),ta_group_id) add_permissions(history_file, stat.S_IRGRP) grading_finished = dateutils.get_current_time() try: shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not copy ",os.path.join(tmp_work,"grade.txt")) log_message(log_path, job_id, is_batch_job, which_untrusted, item_name, message="ERROR: grade.txt does not exist") log_stack_trace(stack_trace_log_path, job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) grade_result = "" try: with open(os.path.join(tmp_work,"grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open ",os.path.join(tmp_work,"grade.txt")) log_message(job_id,is_batch_job,which_untrusted,item_name,message="ERROR: grade.txt does not exist") log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) gradeable_deadline_string = gradeable_config_obj["date_due"] submission_datetime = dateutils.read_submitty_date(submission_string) gradeable_deadline_datetime = dateutils.read_submitty_date(gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date(gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int((submission_datetime-gradeable_deadline_datetime).total_seconds()) # note: negative = not late grading_finished_longstring = dateutils.write_submitty_date(grading_finished) with open(os.path.join(tmp_submission,".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) gradingtime = (grading_finished - grading_began).total_seconds() queue_obj["gradingtime"]=gradingtime queue_obj["grade_result"]=grade_result queue_obj["which_untrusted"]=which_untrusted waittime = queue_obj["waittime"] with open(os.path.join(tmp_results,"queue_file.json"),'w') as outfile: json.dump(queue_obj,outfile,sort_keys=True,indent=4,separators=(',', ': ')) try: shutil.move(os.path.join(tmp_work, "results.json"), os.path.join(tmp_results, "results.json")) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open/write ",os.path.join(tmp_work,"results.json")) log_message(log_path, job_id,is_batch_job,which_untrusted,item_name,message="ERROR: results.json read/write error") log_stack_trace(stack_trace_log_path, job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) just_write_grade_history(history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_obj["queue_time"], "BATCH" if is_batch_job else "INTERACTIVE", grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result, queue_obj.get("revision", None)) with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: f.write("FINISHED GRADING!\n") # save the logs! shutil.copytree(tmp_logs,os.path.join(tmp_results,"logs")) log_message(log_path, job_id,is_batch_job,which_untrusted,item_name,"grade:",gradingtime,grade_result)
def just_grade_item(next_directory,next_to_grade,which_untrusted): my_pid = os.getpid() # verify the hwcron user is running this script if not int(os.getuid()) == int(HWCRON_UID): grade_items_logging.log_message("ERROR: must be run by hwcron") raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user") # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = get_submission_path(next_directory,next_to_grade) submission_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"], "submissions",obj["gradeable"],obj["who"],str(obj["version"])) if not os.path.isdir(submission_path): grade_items_logging.log_message("ERROR: the submission directory does not exist" + submission_path) raise SystemExit("ERROR: the submission directory does not exist",submission_path) print ("pid",my_pid,"GRADE THIS", submission_path) is_vcs,vcs_type,vcs_base_url,vcs_subdirectory = get_vcs_info(SUBMITTY_DATA_DIR,obj["semester"],obj["course"],obj["gradeable"],obj["who"],obj["team"]) is_batch_job = next_directory==BATCH_QUEUE is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory,next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = int((grading_began-queue_time).total_seconds()) grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"wait:",waittime,"") # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"provided_code",obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_input",obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_output",obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"custom_validation_code",obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"bin") checkout_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"checkout",obj["gradeable"],obj["who"],str(obj["version"])) results_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"results",obj["gradeable"],obj["who"],str(obj["version"])) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path,"history.json") history_file_tmp = "" if os.path.isfile(history_file): filehandle,history_file_tmp = tempfile.mkstemp() shutil.copy(history_file,history_file_tmp) # get info from the gradeable config file json_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","form","form_"+obj["gradeable"]+".json") with open(json_config, 'r') as infile: gradeable_config_obj = json.load(infile) # get info from the gradeable config file complete_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","complete_config","complete_config_"+obj["gradeable"]+".json") with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get("use_checkout_subdirectory","") checkout_subdir_path = os.path.join(checkout_path,checkout_subdirectory) # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = os.path.join("/var/local/submitty/autograding_tmp/",which_untrusted,"tmp") shutil.rmtree(tmp,ignore_errors=True) os.makedirs(tmp) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp,"tmp_logs") os.makedirs(tmp_logs) # grab the submission time with open (os.path.join(submission_path,".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) # -------------------------------------------------------------------- # CHECKOUT THE STUDENT's REPO if is_vcs: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nVCS CHECKOUT", file=f) print ("vcs_subdirectory",vcs_subdirectory, file=f) # cleanup the previous checkout (if it exists) shutil.rmtree(checkout_path,ignore_errors=True) os.makedirs(checkout_path, exist_ok=True) subprocess.call (['/usr/bin/git', 'clone', vcs_subdirectory, checkout_path]) os.chdir(checkout_path) # determine which version we need to checkout what_version = subprocess.check_output(['git', 'rev-list', '-n', '1', '--before="'+submission_string+'"', 'master']) what_version = str(what_version.decode('utf-8')).rstrip() if what_version == "": # oops, pressed the grade button before a valid commit shutil.rmtree(checkout_path,ignore_errors=True) else: # and check out the right version subprocess.call (['git', 'checkout', '-b', 'grade', what_version]) os.chdir(tmp) subprocess.call(['ls', '-lR', checkout_path], stdout=open(tmp_logs + "/overall.txt", 'a')) # -------------------------------------------------------------------- # START DOCKER container = None if USE_DOCKER: container = subprocess.check_output(['docker', 'run', '-t', '-d', '-v', tmp + ':' + tmp, 'ubuntu:custom']).decode('utf8').strip() # -------------------------------------------------------------------- # COMPILE THE SUBMITTED CODE with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nCOMPILATION STARTS", file=f) # copy submitted files to the tmp compilation directory tmp_compilation = os.path.join(tmp,"TMP_COMPILATION") os.mkdir(tmp_compilation) os.chdir(tmp_compilation) gradeable_deadline_string = gradeable_config_obj["date_due"] patterns_submission_to_compilation = complete_config_obj["autograding"]["submission_to_compilation"] pattern_copy("submission_to_compilation",patterns_submission_to_compilation,submission_path,tmp_compilation,tmp_logs) if is_vcs: pattern_copy("checkout_to_compilation",patterns_submission_to_compilation,checkout_subdir_path,tmp_compilation,tmp_logs) # copy any instructor provided code files to tmp compilation directory copy_contents_into(provided_code_path,tmp_compilation,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy compile.out to the current directory shutil.copy (os.path.join(bin_path,obj["gradeable"],"compile.out"),os.path.join(tmp_compilation,"my_compile.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_compilation, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP) add_permissions(tmp,stat.S_IROTH | stat.S_IXOTH) add_permissions(tmp_logs,stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) with open(os.path.join(tmp_logs,"compilation_log.txt"), 'w') as logfile: if USE_DOCKER: compile_success = subprocess.call(['docker', 'exec', '-w', tmp_compilation, container, os.path.join(tmp_compilation, 'my_compile.out'), obj['gradeable'], obj['who'], str(obj['version']), submission_string], stdout=logfile) else: compile_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"), which_untrusted, os.path.join(tmp_compilation,"my_compile.out"), obj["gradeable"], obj["who"], str(obj["version"]), submission_string], stdout=logfile) if compile_success == 0: print ("pid",my_pid,"COMPILATION OK") else: print ("pid",my_pid,"COMPILATION FAILURE") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","COMPILATION FAILURE") #raise SystemExit() untrusted_grant_rwx_access(which_untrusted,tmp_compilation) # remove the compilation program os.remove(os.path.join(tmp_compilation,"my_compile.out")) # return to the main tmp directory os.chdir(tmp) # -------------------------------------------------------------------- # make the runner directory with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nRUNNER STARTS", file=f) tmp_work = os.path.join(tmp,"TMP_WORK") os.makedirs(tmp_work) os.chdir(tmp_work) # move all executable files from the compilation directory to the main tmp directory # Note: Must preserve the directory structure of compiled files (esp for Java) patterns_submission_to_runner = complete_config_obj["autograding"]["submission_to_runner"] pattern_copy("submission_to_runner",patterns_submission_to_runner,submission_path,tmp_work,tmp_logs) if is_vcs: pattern_copy("checkout_to_runner",patterns_submission_to_runner,checkout_subdir_path,tmp_work,tmp_logs) patterns_compilation_to_runner = complete_config_obj["autograding"]["compilation_to_runner"] pattern_copy("compilation_to_runner",patterns_compilation_to_runner,tmp_compilation,tmp_work,tmp_logs) # copy input files to tmp_work directory copy_contents_into(test_input_path,tmp_work,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy runner.out to the current directory shutil.copy (os.path.join(bin_path,obj["gradeable"],"run.out"),os.path.join(tmp_work,"my_runner.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # raise SystemExit() # run the run.out as the untrusted user with open(os.path.join(tmp_logs,"runner_log.txt"), 'w') as logfile: if USE_DOCKER: runner_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_runner.out'), obj['gradeable'], obj['who'], str(obj['version']), submission_string], stdout=logfile) else: runner_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"), which_untrusted, os.path.join(tmp_work,"my_runner.out"), obj["gradeable"], obj["who"], str(obj["version"]), submission_string], stdout=logfile) if runner_success == 0: print ("pid",my_pid,"RUNNER OK") else: print ("pid",my_pid,"RUNNER FAILURE") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","RUNNER FAILURE") untrusted_grant_rwx_access(which_untrusted,tmp_work) untrusted_grant_rwx_access(which_untrusted,tmp_compilation) # -------------------------------------------------------------------- # RUN VALIDATOR with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nVALIDATION STARTS", file=f) # copy results files from compilation... patterns_submission_to_validation = complete_config_obj["autograding"]["submission_to_validation"] pattern_copy("submission_to_validation",patterns_submission_to_validation,submission_path,tmp_work,tmp_logs) if is_vcs: pattern_copy("checkout_to_validation",patterns_submission_to_validation,checkout_subdir_path,tmp_work,tmp_logs) patterns_compilation_to_validation = complete_config_obj["autograding"]["compilation_to_validation"] pattern_copy("compilation_to_validation",patterns_compilation_to_validation,tmp_compilation,tmp_work,tmp_logs) # remove the compilation directory shutil.rmtree(tmp_compilation) # copy output files to tmp_work directory copy_contents_into(test_output_path,tmp_work,tmp_logs) # copy any instructor custom validation code into the tmp work directory copy_contents_into(custom_validation_code_path,tmp_work,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy validator.out to the current directory shutil.copy (os.path.join(bin_path,obj["gradeable"],"validate.out"),os.path.join(tmp_work,"my_validator.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(os.path.join(tmp_work,"my_validator.out"),stat.S_IROTH | stat.S_IXOTH) # validator the validator.out as the untrusted user with open(os.path.join(tmp_logs,"validator_log.txt"), 'w') as logfile: if USE_DOCKER: validator_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_validator.out'), obj['gradeable'], obj['who'], str(obj['version']), submission_string], stdout=logfile) else: validator_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"), which_untrusted, os.path.join(tmp_work,"my_validator.out"), obj["gradeable"], obj["who"], str(obj["version"]), submission_string], stdout=logfile) if validator_success == 0: print ("pid",my_pid,"VALIDATOR OK") else: print ("pid",my_pid,"VALIDATOR FAILURE") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","VALIDATION FAILURE") untrusted_grant_rwx_access(which_untrusted,tmp_work) # grab the result of autograding grade_result = "" with open(os.path.join(tmp_work,"grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line # -------------------------------------------------------------------- # MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nARCHIVING STARTS", file=f) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) os.chdir(bin_path) # save the old results path! if os.path.isdir(os.path.join(results_path,"OLD")): shutil.move(os.path.join(results_path,"OLD"), os.path.join(tmp,"OLD_RESULTS")) # clean out all of the old files if this is a re-run shutil.rmtree(results_path,ignore_errors=True) # create the directory (and the full path if it doesn't already exist) os.makedirs(results_path) # bring back the old results! if os.path.isdir(os.path.join(tmp,"OLD_RESULTS")): shutil.move(os.path.join(tmp,"OLD_RESULTS"), os.path.join(results_path,"OLD")) os.makedirs(os.path.join(results_path,"details")) patterns_work_to_details = complete_config_obj["autograding"]["work_to_details"] pattern_copy("work_to_details",patterns_work_to_details,tmp_work,os.path.join(results_path,"details"),tmp_logs) if not history_file_tmp == "": shutil.move(history_file_tmp,history_file) # fix permissions ta_group_id = os.stat(results_path).st_gid os.chown(history_file,int(HWCRON_UID),ta_group_id) add_permissions(history_file,stat.S_IRGRP) grading_finished = dateutils.get_current_time() shutil.copy(os.path.join(tmp_work,"results.json"),results_path) shutil.copy(os.path.join(tmp_work,"grade.txt"),results_path) # ------------------------------------------------------------- # create/append to the results history gradeable_deadline_datetime = dateutils.read_submitty_date(gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date(gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int((submission_datetime-gradeable_deadline_datetime).total_seconds()) # note: negative = not late grading_began_longstring = dateutils.write_submitty_date(grading_began) grading_finished_longstring = dateutils.write_submitty_date(grading_finished) gradingtime = int((grading_finished-grading_began).total_seconds()) write_grade_history.just_write_grade_history(history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_time_longstring, is_batch_job_string, grading_began_longstring, waittime, grading_finished_longstring, gradingtime, grade_result) #--------------------------------------------------------------------- # WRITE OUT VERSION DETAILS if WRITE_DATABASE: insert_database_version_data.insert_to_database( obj["semester"], obj["course"], obj["gradeable"], obj["user"], obj["team"], obj["who"], True if obj["is_team"] else False, str(obj["version"])) print ("pid",my_pid,"finished grading ", next_to_grade, " in ", gradingtime, " seconds") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"grade:",gradingtime,grade_result) with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: f.write("FINISHED GRADING!") # save the logs! shutil.copytree(tmp_logs,os.path.join(results_path,"logs")) # -------------------------------------------------------------------- # REMOVE TEMP DIRECTORY shutil.rmtree(tmp) # -------------------------------------------------------------------- # CLEAN UP DOCKER if USE_DOCKER: subprocess.call(['docker', 'rm', '-f', container])
def get_job(my_name,which_machine,my_capabilities,which_untrusted,overall_lock): """ Picks a job from the queue :param overall_lock: a lock on the directory containing all queue files """ time_get_job_begin = dateutils.get_current_time() overall_lock.acquire() folder= INTERACTIVE_QUEUE # ---------------------------------------------------------------- # Our first priority is to perform any awaiting VCS checkouts # Note: This design is imperfect: # # * If all shippers are busy working on long-running autograding # tasks there will be a delay of seconds or minutes between # a student pressing the submission button and clone happening. # This is a minor exploit allowing them to theoretically # continue working on their submission past the deadline for # the time period of the delay. # -- This is not a significant, practical problem. # # * If multiple and/or large git submissions arrive close # together, this shipper job will be tied up performing these # clone operations. Because we don't release the lock, any # other shippers that complete their work will also be blocked # from either helping with the clones or tackling the next # autograding job. # -- Based on experience with actual submission patterns, we # do not anticipate that this will be a significant # bottleneck at this time. # # * If a git clone takes a very long time and/or hangs because of # network problems, this could halt all work on the server. # -- We'll need to monitor the production server. # # We plan to do a complete overhaul of the # scheduler/shipper/worker and refactoring this design should be # part of the project. # Grab all the VCS files currently in the folder... vcs_files = [str(f) for f in Path(folder).glob('VCS__*')] for f in vcs_files: vcs_file = f[len(folder)+1:] no_vcs_file = f[len(folder)+1+5:] # do the checkout updated_obj = checkout_vcs_repo(folder+"/"+vcs_file) # save the regular grading queue file with open(os.path.join(folder,no_vcs_file), "w") as queue_file: json.dump(updated_obj, queue_file) # cleanup the vcs queue file os.remove(folder+"/"+vcs_file) # ---------------------------------------------------------------- # Grab all the files currently in the folder, sorted by creation # time, and put them in the queue to be graded files = [str(f) for f in Path(folder).glob('*')] files_and_times = list() for f in files: try: my_time = os.path.getctime(f) except: continue tup = (f, my_time) files_and_times.append(tup) files_and_times = sorted(files_and_times, key=operator.itemgetter(1)) my_job="" for full_path_file, file_time in files_and_times: # get the file name (without the path) just_file = full_path_file[len(folder)+1:] # skip items that are already being graded if (just_file[0:8]=="GRADING_"): continue grading_file = os.path.join(folder,"GRADING_"+just_file) if grading_file in files: continue # found something to do try: with open(full_path_file, 'r') as infile: queue_obj = json.load(infile) except: continue #Check to make sure that we are capable of grading this submission required_capabilities = queue_obj["required_capabilities"] if not required_capabilities in my_capabilities: continue # prioritize interactive jobs over (batch) regrades # if you've found an interactive job, exit early (since they are sorted by timestamp) if not "regrade" in queue_obj or not queue_obj["regrade"]: my_job = just_file break # otherwise it's a regrade, and if we don't already have a # job, take it, but we have to search the rest of the list if my_job == "": my_job = just_file if not my_job == "": grading_file = os.path.join(folder, "GRADING_" + my_job) # create the grading file with open(os.path.join(grading_file), "w") as queue_file: json.dump({"untrusted": which_untrusted}, queue_file) overall_lock.release() time_get_job_end = dateutils.get_current_time() time_delta = time_get_job_end-time_get_job_begin if time_delta > datetime.timedelta(milliseconds=100): print (my_name, " WARNING: submitty_autograding shipper get_job time ", time_delta) grade_items_logging.log_message(JOB_ID, message=str(my_name)+" WARNING: submitty_autograding shipper get_job time "+str(time_delta)) return (my_job)
def prepare_autograding_and_submission_zip( which_machine, which_untrusted, next_directory, next_to_grade ): os.chdir(SUBMITTY_DATA_DIR) # generate a random id to be used to track this job in the autograding logs job_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(6)) # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = load_queue_file_obj(job_id, next_directory, next_to_grade) # The top level course directory for this class course_dir = os.path.join(SUBMITTY_DATA_DIR, 'courses', obj["semester"], obj["course"]) if "generate_output" not in obj: partial_path = os.path.join(obj["gradeable"], obj["who"], str(obj["version"])) item_name = os.path.join(obj["semester"], obj["course"], "submissions", partial_path) submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name) if not os.path.isdir(submission_path): autograding_utils.log_message( AUTOGRADING_LOG_PATH, job_id, message=f"ERROR: the submission directory does not exist {submission_path}" ) raise RuntimeError("ERROR: the submission directory does not exist", submission_path) print(which_machine, which_untrusted, "prepare zip", submission_path) is_vcs, vcs_type, vcs_base_url, vcs_subdirectory = get_vcs_info( SUBMITTY_DATA_DIR, obj["semester"], obj["course"], obj["gradeable"], obj["who"], obj["team"] ) elif obj["generate_output"]: item_name = os.path.join( obj["semester"], obj["course"], "generated_output", obj["gradeable"] ) is_batch_job = "regrade" in obj and obj["regrade"] queue_time = get_queue_time(next_directory, next_to_grade) grading_began = dateutils.get_current_time() waittime = (grading_began-queue_time).total_seconds() autograding_utils.log_message( AUTOGRADING_LOG_PATH, job_id, is_batch_job, "zip", item_name, "wait:", waittime, "" ) # -------------------------------------------------------- # various paths provided_code_path = os.path.join(course_dir, "provided_code", obj["gradeable"]) instructor_solution_path = os.path.join(course_dir, "instructor_solution", obj["gradeable"]) test_input_path = os.path.join(course_dir, "test_input", obj["gradeable"]) test_output_path = os.path.join(course_dir, "test_output", obj["gradeable"]) bin_path = os.path.join(course_dir, "bin", obj["gradeable"]) form_json_config = os.path.join(course_dir, "config", "form", f"form_{obj['gradeable']}.json") custom_validation_code_path = os.path.join( course_dir, "custom_validation_code", obj["gradeable"] ) generated_output_path = os.path.join( course_dir, "generated_output", obj["gradeable"], "random_output" ) complete_config = os.path.join( course_dir, "config", "complete_config", f"complete_config_{obj['gradeable']}.json" ) if not os.path.exists(form_json_config): autograding_utils.log_message( AUTOGRADING_LOG_PATH, job_id, message=f"ERROR: the form json file does not exist {form_json_config}" ) raise RuntimeError(f"ERROR: the form json file does not exist {form_json_config}") if not os.path.exists(complete_config): autograding_utils.log_message( AUTOGRADING_LOG_PATH, job_id, message=f"ERROR: the complete config file does not exist {complete_config}" ) raise RuntimeError(f"ERROR: the complete config file does not exist {complete_config}") # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = tempfile.mkdtemp() tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") os.mkdir(tmp_autograding) tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") os.mkdir(tmp_submission) copytree_if_exists(provided_code_path, os.path.join(tmp_autograding, "provided_code")) copytree_if_exists(test_input_path, os.path.join(tmp_autograding, "test_input")) copytree_if_exists(test_output_path, os.path.join(tmp_autograding, "test_output")) copytree_if_exists(generated_output_path, os.path.join(tmp_autograding, "generated_output")) copytree_if_exists(bin_path, os.path.join(tmp_autograding, "bin")) copytree_if_exists( instructor_solution_path, os.path.join(tmp_autograding, "instructor_solution") ) copytree_if_exists( custom_validation_code_path, os.path.join(tmp_autograding, "custom_validation_code") ) # Copy the default submitty_router into bin. router_path = os.path.join( SUBMITTY_INSTALL_DIR, 'src', 'grading', 'python', 'submitty_router.py' ) shutil.copy(router_path, os.path.join(tmp_autograding, "bin")) shutil.copy(form_json_config, os.path.join(tmp_autograding, "form.json")) shutil.copy(complete_config, os.path.join(tmp_autograding, "complete_config.json")) if "generate_output" not in obj: checkout_path = os.path.join(course_dir, "checkout", partial_path) results_path = os.path.join(course_dir, "results", partial_path) elif obj["generate_output"]: results_path = os.path.join(course_dir, "generated_output", obj["gradeable"]) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path, "history.json") if os.path.isfile(history_file): shutil.copy(history_file, os.path.join(tmp_submission, "history.json")) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") os.makedirs(tmp_logs) # 'touch' a file in the logs folder open(os.path.join(tmp_logs, "overall.txt"), 'a') # -------------------------------------------------------------------- # CONFIRM WE HAVE A CHECKOUT OF THE STUDENT'S REPO if "generate_output" not in obj: if is_vcs: # there should be a checkout log file in the results directory # move that file to the tmp logs directory. vcs_checkout_logfile = os.path.join(results_path, "logs", "vcs_checkout.txt") if os.path.isfile(vcs_checkout_logfile): shutil.move(vcs_checkout_logfile, tmp_logs) else: autograding_utils.log_message( AUTOGRADING_LOG_PATH, job_id, message=f"ERROR: missing vcs_checkout.txt logfile {str(vcs_checkout_logfile)}" ) if "generate_output" not in obj: copytree_if_exists(submission_path, os.path.join(tmp_submission, "submission")) copytree_if_exists(checkout_path, os.path.join(tmp_submission, "checkout")) obj["queue_time"] = dateutils.write_submitty_date(queue_time) obj["regrade"] = is_batch_job obj["waittime"] = waittime obj["job_id"] = job_id with open(os.path.join(tmp_submission, "queue_file.json"), 'w') as outfile: json.dump(obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) user_assignment_access_json = os.path.join( SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "submissions", obj["gradeable"], obj["who"], "user_assignment_access.json") user_assignment_settings_json = os.path.join( SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "submissions", obj["gradeable"], obj["who"], "user_assignment_settings.json") if os.path.exists(user_assignment_access_json): shutil.copy( user_assignment_access_json, os.path.join(tmp_submission, "user_assignment_access.json") ) if os.path.exists(user_assignment_settings_json): shutil.copy( user_assignment_settings_json, os.path.join(tmp_submission, "user_assignment_settings.json") ) grading_began_longstring = dateutils.write_submitty_date(grading_began) with open(os.path.join(tmp_submission, ".grading_began"), 'w') as f: print(grading_began_longstring, file=f) # zip up autograding & submission folders filehandle1, my_autograding_zip_file = tempfile.mkstemp() filehandle2, my_submission_zip_file = tempfile.mkstemp() autograding_utils.zip_my_directory(tmp_autograding, my_autograding_zip_file) autograding_utils.zip_my_directory(tmp_submission, my_submission_zip_file) os.close(filehandle1) os.close(filehandle2) # cleanup shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp) return (my_autograding_zip_file, my_submission_zip_file)
def get_job(my_name,which_machine,my_capabilities,which_untrusted,overall_lock): """ Picks a job from the queue :param overall_lock: a lock on the directory containing all queue files """ time_get_job_begin = dateutils.get_current_time() overall_lock.acquire() folder= INTERACTIVE_QUEUE # Grab all the files currently in the folder, sorted by creation # time, and put them in the queue to be graded files = glob.glob(os.path.join(folder, "*")) files_and_times = list() for f in files: try: my_time = os.path.getctime(f) except: continue tup = (f, my_time) files_and_times.append(tup) files_and_times = sorted(files_and_times, key=operator.itemgetter(1)) my_job="" for full_path_file, file_time in files_and_times: # get the file name (without the path) just_file = full_path_file[len(folder)+1:] # skip items that are already being graded if (just_file[0:8]=="GRADING_"): continue grading_file = os.path.join(folder,"GRADING_"+just_file) if grading_file in files: continue # found something to do try: with open(full_path_file, 'r') as infile: queue_obj = json.load(infile) except: continue #Check to make sure that we are capable of grading this submission required_capabilities = queue_obj["required_capabilities"] if not required_capabilities in my_capabilities: continue # prioritize interactive jobs over (batch) regrades # if you've found an interactive job, exit early (since they are sorted by timestamp) if not "regrade" in queue_obj or not queue_obj["regrade"]: my_job = just_file break # otherwise it's a regrade, and if we don't already have a # job, take it, but we have to search the rest of the list if my_job == "": my_job = just_file if not my_job == "": grading_file = os.path.join(folder, "GRADING_" + my_job) # create the grading file with open(os.path.join(grading_file), "w") as queue_file: json.dump({"untrusted": which_untrusted}, queue_file) overall_lock.release() time_get_job_end = dateutils.get_current_time() time_delta = time_get_job_end-time_get_job_begin if time_delta > datetime.timedelta(milliseconds=100): print (my_name, " WARNING: submitty_autograding shipper get_job time ", time_delta) grade_items_logging.log_message(JOB_ID, message=str(my_name)+" WARNING: submitty_autograding shipper get_job time "+str(time_delta)) return (my_job)
def archive_autograding_results(config, working_directory: os.PathLike, job_id: str, which_untrusted: str, is_batch_job: bool, complete_config_obj: dict, gradeable_config_obj: dict, queue_obj: dict, is_test_environment: bool): """ After grading is finished, archive the results. """ tmp_autograding = os.path.join(working_directory, "TMP_AUTOGRADING") tmp_submission = os.path.join(working_directory, "TMP_SUBMISSION") tmp_work = os.path.join(working_directory, "TMP_WORK") tmp_logs = os.path.join(working_directory, "TMP_SUBMISSION", "tmp_logs") tmp_results = os.path.join(working_directory, "TMP_RESULTS") submission_path = os.path.join(tmp_submission, "submission") random_output_path = os.path.join(tmp_work, 'random_output') if "generate_output" not in queue_obj: partial_path = os.path.join(queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"], queue_obj["course"], "submissions", partial_path) elif queue_obj["generate_output"]: item_name = os.path.join(queue_obj["semester"], queue_obj["course"], "generated_output", queue_obj["gradeable"]) results_public_dir = os.path.join(tmp_results, "results_public") results_details_dir = os.path.join(tmp_results, "details") patterns = complete_config_obj['autograding'] # Copy work to details pattern_copy("work_to_details", patterns['work_to_details'], tmp_work, results_details_dir, tmp_logs) # Copy work to public if 'work_to_public' in patterns: pattern_copy("work_to_public", patterns['work_to_public'], tmp_work, results_public_dir, tmp_logs) if os.path.exists(random_output_path): pattern_copy("work_to_random_output", [ os.path.join(random_output_path, '**', '*.txt'), ], tmp_work, tmp_results, tmp_logs) # timestamp of first access to the gradeable page first_access_string = "" # grab the submission time if "generate_output" in queue_obj and queue_obj["generate_output"]: submission_string = "" else: with open( os.path.join(tmp_submission, 'submission', ".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() # grab the first access to the gradeable page (if it exists) user_assignment_access_filename = os.path.join( tmp_submission, ".user_assignment_access.json") if os.path.exists(user_assignment_access_filename): with open(user_assignment_access_filename, 'r') as access_file: obj = json.load(access_file) first_access_string = obj[0]["timestamp"] history_file_tmp = os.path.join(tmp_submission, "history.json") history_file = os.path.join(tmp_results, "history.json") if os.path.isfile(history_file_tmp) and not is_test_environment: shutil.move(history_file_tmp, history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file, int(config.submitty_users['daemon_uid']), ta_group_id) add_permissions(history_file, stat.S_IRGRP) grading_finished = dateutils.get_current_time() grade_result = "" if "generate_output" not in queue_obj: try: shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results) with open(os.path.join(tmp_work, "grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line except Exception as e: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: f.write( f"\n\nERROR: Grading incomplete -- Could not process {os.path.join(tmp_work,'grade.txt')}" ) config.logger.log_message( "ERROR: could not process grade.txt. See stack trace entry for more details.", job_id=job_id, is_batch=is_batch_job, which_untrusted=which_untrusted, jobname=item_name, ) config.logger.log_stack_trace( traceback.format_exc(), job_id=job_id, is_batch=is_batch_job, which_untrusted=which_untrusted, jobname=item_name, ) submission_datetime = dateutils.read_submitty_date(submission_string) submission_longstring = dateutils.write_submitty_date( submission_datetime) # compute lateness (if there is a due date / submission deadline) gradeable_deadline_string = gradeable_config_obj["date_due"] if gradeable_deadline_string is None: print("NO DEADLINE") gradeable_deadline_longstring = "None" seconds_late = 0 else: print("DEADLINE IS '" + str(gradeable_deadline_string) + "'") gradeable_deadline_datetime = dateutils.read_submitty_date( gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date( gradeable_deadline_datetime) seconds_late = int((submission_datetime - gradeable_deadline_datetime).total_seconds()) # compute the access duration in seconds (if it exists) access_duration = -1 if first_access_string != "": first_access_datetime = dateutils.read_submitty_date( first_access_string) access_duration = int( (submission_datetime - first_access_datetime).total_seconds()) # note: negative = not late grading_finished_longstring = dateutils.write_submitty_date( grading_finished) with open(os.path.join(tmp_submission, ".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) gradingtime = (grading_finished - grading_began).total_seconds() queue_obj["gradingtime"] = gradingtime queue_obj["grade_result"] = grade_result queue_obj["which_untrusted"] = which_untrusted waittime = queue_obj["waittime"] try: # Make certain results.json is utf-8 encoded. results_json_path = os.path.join(tmp_work, 'results.json') with codecs.open(results_json_path, 'r', encoding='utf-8', errors='ignore') as infile: results_str = "".join(line.rstrip() for line in infile) results_obj = json.loads(results_str) with open(results_json_path, 'w') as outfile: json.dump(results_obj, outfile, indent=4) shutil.move(results_json_path, os.path.join(tmp_results, "results.json")) except Exception as e: with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: f.write( f"\n\nERROR: Grading incomplete -- Could not open/write {os.path.join(tmp_work,'results.json')}" ) config.logger.log_message( "ERROR: results.json read/write error", job_id=job_id, is_batch=is_batch_job, which_untrusted=which_untrusted, jobname=item_name, ) config.logger.log_stack_trace( traceback.format_exc(), job_id=job_id, is_batch=is_batch_job, which_untrusted=which_untrusted, jobname=item_name, ) # Rescue custom validator files custom_validator_output_directory = os.path.join( tmp_results, "custom_validator_output") pattern_copy("rescue_custom_validator_validation_jsons", [ os.path.join(tmp_work, 'validation_results_*.json'), ], tmp_work, custom_validator_output_directory, tmp_logs) pattern_copy("rescue_custom_validator_logs", [ os.path.join(tmp_work, 'validation_logfile_*.txt'), ], tmp_work, custom_validator_output_directory, tmp_logs) pattern_copy("rescue_custom_validator_errors", [ os.path.join(tmp_work, 'validation_stderr_*.txt'), ], tmp_work, custom_validator_output_directory, tmp_logs) just_write_grade_history(history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, first_access_string, access_duration, queue_obj["queue_time"], "BATCH" if is_batch_job else "INTERACTIVE", grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result, queue_obj.get("revision", None)) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: f.write("FINISHED GRADING!\n") config.logger.log_message(grade_result, job_id=job_id, is_batch=is_batch_job, which_untrusted=which_untrusted, jobname=item_name, timelabel="grade:", elapsed_time=gradingtime) with open(os.path.join(tmp_results, "queue_file.json"), 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) # save the logs! shutil.copytree(tmp_logs, os.path.join(tmp_results, "logs")) # Save the .submit.notebook # Copy the .submit.notebook to tmp_work for validation submit_notebook_path = os.path.join(tmp_submission, 'submission', ".submit.notebook") if os.path.exists(submit_notebook_path): shutil.copy(submit_notebook_path, os.path.join(tmp_results, ".submit.notebook"))
def archive_autograding_results(working_directory, job_id, which_untrusted, is_batch_job, complete_config_obj, gradeable_config_obj, queue_obj, log_path, stack_trace_log_path, is_test_environment): """ After grading is finished, archive the results. """ tmp_autograding = os.path.join(working_directory,"TMP_AUTOGRADING") tmp_submission = os.path.join(working_directory,"TMP_SUBMISSION") tmp_work = os.path.join(working_directory,"TMP_WORK") tmp_logs = os.path.join(working_directory,"TMP_SUBMISSION","tmp_logs") tmp_results = os.path.join(working_directory,"TMP_RESULTS") submission_path = os.path.join(tmp_submission, "submission") random_output_path = os.path.join(tmp_work, 'random_output') if "generate_output" not in queue_obj: partial_path = os.path.join(queue_obj["gradeable"],queue_obj["who"],str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"],queue_obj["course"],"submissions",partial_path) elif queue_obj["generate_output"]: item_name = os.path.join(queue_obj["semester"],queue_obj["course"],"generated_output",queue_obj["gradeable"]) results_public_dir = os.path.join(tmp_results,"results_public") results_details_dir = os.path.join(tmp_results, "details") patterns = complete_config_obj['autograding'] # Copy work to details pattern_copy("work_to_details", patterns['work_to_details'], tmp_work, results_details_dir, tmp_logs) # Copy work to public if 'work_to_public' in patterns: pattern_copy("work_to_public", patterns['work_to_public'], tmp_work, results_public_dir, tmp_logs) if os.path.exists(random_output_path): pattern_copy("work_to_random_output", [os.path.join(random_output_path, 'test*', '**', '*.txt'),], tmp_work, tmp_results, tmp_logs) # timestamp of first access to the gradeable page first_access_string = "" # grab the submission time if "generate_output" in queue_obj and queue_obj["generate_output"]: submission_string = "" else: with open(os.path.join(tmp_submission, 'submission' ,".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() # grab the first access to the gradeable page (if it exists) user_assignment_access_filename = os.path.join(tmp_submission, "user_assignment_access.json") if os.path.exists(user_assignment_access_filename): with open(user_assignment_access_filename, 'r') as access_file: obj = json.load(access_file, object_pairs_hook=collections.OrderedDict) first_access_string = obj["page_load_history"][0]["time"] history_file_tmp = os.path.join(tmp_submission,"history.json") history_file = os.path.join(tmp_results,"history.json") if os.path.isfile(history_file_tmp) and not is_test_environment: from . import CONFIG_PATH with open(os.path.join(CONFIG_PATH, 'submitty_users.json')) as open_file: OPEN_JSON = json.load(open_file) DAEMON_UID = OPEN_JSON['daemon_uid'] shutil.move(history_file_tmp, history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file, int(DAEMON_UID),ta_group_id) add_permissions(history_file, stat.S_IRGRP) grading_finished = dateutils.get_current_time() if "generate_output" not in queue_obj: try: shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not copy ",os.path.join(tmp_work,"grade.txt")) log_message(log_path, job_id, is_batch_job, which_untrusted, item_name, message="ERROR: grade.txt does not exist") log_stack_trace(stack_trace_log_path, job_id, is_batch_job, which_untrusted, item_name, trace=traceback.format_exc()) grade_result = "" try: with open(os.path.join(tmp_work,"grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open ",os.path.join(tmp_work,"grade.txt")) log_message(job_id,is_batch_job,which_untrusted,item_name,message="ERROR: grade.txt does not exist") log_stack_trace(job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) gradeable_deadline_string = gradeable_config_obj["date_due"] # FIXME: The access date string is currently misformatted # mm-dd-yyyy, but we want yyyy-mm-dd. Also it is missing # the common name timezone string, e.g., "America/NewYork". # We should standardize this logging eventually, but # keeping it as is because we are mid-semester with this # new feature and I don't want to break things. first_access_string = dateutils.normalize_submitty_date(first_access_string) submission_datetime = dateutils.read_submitty_date(submission_string) gradeable_deadline_datetime = dateutils.read_submitty_date(gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date(gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int((submission_datetime-gradeable_deadline_datetime).total_seconds()) # compute the access duration in seconds (if it exists) access_duration = -1 if first_access_string != "": first_access_datetime = dateutils.read_submitty_date(first_access_string) access_duration = int((submission_datetime-first_access_datetime).total_seconds()) # note: negative = not late grading_finished_longstring = dateutils.write_submitty_date(grading_finished) with open(os.path.join(tmp_submission,".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) gradingtime = (grading_finished - grading_began).total_seconds() queue_obj["gradingtime"]=gradingtime queue_obj["grade_result"]=grade_result queue_obj["which_untrusted"]=which_untrusted waittime = queue_obj["waittime"] try: # Make certain results.json is utf-8 encoded. results_json_path = os.path.join(tmp_work, 'results.json') with codecs.open(results_json_path, 'r', encoding='utf-8', errors='ignore') as infile: results_str = "".join(line.rstrip() for line in infile) results_obj = json.loads(results_str) with open(results_json_path, 'w') as outfile: json.dump(results_obj, outfile, indent=4) shutil.move(results_json_path, os.path.join(tmp_results, "results.json")) except: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\n\nERROR: Grading incomplete -- Could not open/write ",os.path.join(tmp_work,"results.json")) log_message(log_path, job_id,is_batch_job,which_untrusted,item_name,message="ERROR: results.json read/write error") log_stack_trace(stack_trace_log_path, job_id,is_batch_job,which_untrusted,item_name,trace=traceback.format_exc()) # Rescue custom validator files custom_validator_output_directory = os.path.join(tmp_results, "custom_validator_output") pattern_copy("rescue_custom_validator_validation_jsons", [os.path.join(tmp_work, 'validation_results_*.json'),], tmp_work, custom_validator_output_directory, tmp_logs) pattern_copy("rescue_custom_validator_logs", [os.path.join(tmp_work, 'validation_logfile_*.txt'),], tmp_work, custom_validator_output_directory, tmp_logs) pattern_copy("rescue_custom_validator_errors", [os.path.join(tmp_work, 'validation_stderr_*.txt'),], tmp_work, custom_validator_output_directory, tmp_logs) just_write_grade_history(history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, first_access_string, access_duration, queue_obj["queue_time"], "BATCH" if is_batch_job else "INTERACTIVE", grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result, queue_obj.get("revision", None)) with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: f.write("FINISHED GRADING!\n") log_message(log_path, job_id,is_batch_job,which_untrusted,item_name,"grade:",gradingtime,grade_result) with open(os.path.join(tmp_results,"queue_file.json"),'w') as outfile: json.dump(queue_obj,outfile,sort_keys=True,indent=4,separators=(',', ': ')) # save the logs! shutil.copytree(tmp_logs,os.path.join(tmp_results,"logs"))
def prepare_autograding_and_submission_zip(which_machine,which_untrusted,next_directory,next_to_grade): os.chdir(SUBMITTY_DATA_DIR) # generate a random id to be used to track this job in the autograding logs job_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(6)) # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = load_queue_file_obj(job_id,next_directory,next_to_grade) partial_path = os.path.join(obj["gradeable"],obj["who"],str(obj["version"])) item_name = os.path.join(obj["semester"],obj["course"],"submissions",partial_path) submission_path = os.path.join(SUBMITTY_DATA_DIR,"courses",item_name) if not os.path.isdir(submission_path): grade_items_logging.log_message(job_id, message="ERROR: the submission directory does not exist " + submission_path) raise RuntimeError("ERROR: the submission directory does not exist", submission_path) print(which_machine,which_untrusted,"prepare zip",submission_path) is_vcs,vcs_type,vcs_base_url,vcs_subdirectory = get_vcs_info(SUBMITTY_DATA_DIR,obj["semester"],obj["course"],obj["gradeable"],obj["who"],obj["team"]) is_batch_job = "regrade" in obj and obj["regrade"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory,next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = (grading_began-queue_time).total_seconds() grade_items_logging.log_message(job_id,is_batch_job,"zip",item_name,"wait:",waittime,"") # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"provided_code",obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_input",obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_output",obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"custom_validation_code",obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"bin",obj["gradeable"]) form_json_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","form","form_"+obj["gradeable"]+".json") complete_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","complete_config","complete_config_"+obj["gradeable"]+".json") if not os.path.exists(form_json_config): grade_items_logging.log_message(job_id,message="ERROR: the form json file does not exist " + form_json_config) raise RuntimeError("ERROR: the form json file does not exist ",form_json_config) if not os.path.exists(complete_config): grade_items_logging.log_message(job_id,message="ERROR: the complete config file does not exist " + complete_config) raise RuntimeError("ERROR: the complete config file does not exist ",complete_config) # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = tempfile.mkdtemp() tmp_autograding = os.path.join(tmp,"TMP_AUTOGRADING") os.mkdir(tmp_autograding) tmp_submission = os.path.join(tmp,"TMP_SUBMISSION") os.mkdir(tmp_submission) copytree_if_exists(provided_code_path,os.path.join(tmp_autograding,"provided_code")) copytree_if_exists(test_input_path,os.path.join(tmp_autograding,"test_input")) copytree_if_exists(test_output_path,os.path.join(tmp_autograding,"test_output")) copytree_if_exists(custom_validation_code_path,os.path.join(tmp_autograding,"custom_validation_code")) copytree_if_exists(bin_path,os.path.join(tmp_autograding,"bin")) shutil.copy(form_json_config,os.path.join(tmp_autograding,"form.json")) shutil.copy(complete_config,os.path.join(tmp_autograding,"complete_config.json")) checkout_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"checkout",partial_path) results_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"results",partial_path) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path,"history.json") history_file_tmp = "" if os.path.isfile(history_file): shutil.copy(history_file,os.path.join(tmp_submission,"history.json")) # get info from the gradeable config file with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get("use_checkout_subdirectory","") checkout_subdir_path = os.path.join(checkout_path,checkout_subdirectory) queue_file = os.path.join(next_directory,next_to_grade) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp,"TMP_SUBMISSION","tmp_logs") os.makedirs(tmp_logs) # 'touch' a file in the logs folder open(os.path.join(tmp_logs,"overall.txt"), 'a') # -------------------------------------------------------------------- # CONFIRM WE HAVE A CHECKOUT OF THE STUDENT'S REPO if is_vcs: # there should be a checkout log file in the results directory # move that file to the tmp logs directory.. vcs_checkout_logfile = os.path.join(results_path,"logs","vcs_checkout.txt") if os.path.isfile(vcs_checkout_logfile): shutil.move(vcs_checkout_logfile,tmp_logs) else: grade_items_logging.log_message(JOB_ID, message=str(my_name)+" ERROR: missing vcs_checkout.txt logfile "+str(vcs_checkout_logfile)) copytree_if_exists(submission_path,os.path.join(tmp_submission,"submission")) copytree_if_exists(checkout_path,os.path.join(tmp_submission,"checkout")) obj["queue_time"] = queue_time_longstring obj["regrade"] = is_batch_job obj["waittime"] = waittime obj["job_id"] = job_id with open(os.path.join(tmp_submission,"queue_file.json"),'w') as outfile: json.dump(obj,outfile,sort_keys=True,indent=4,separators=(',', ': ')) grading_began_longstring = dateutils.write_submitty_date(grading_began) with open(os.path.join(tmp_submission,".grading_began"), 'w') as f: print (grading_began_longstring,file=f) # zip up autograding & submission folders filehandle1, my_autograding_zip_file =tempfile.mkstemp() filehandle2, my_submission_zip_file =tempfile.mkstemp() grade_item.zip_my_directory(tmp_autograding,my_autograding_zip_file) grade_item.zip_my_directory(tmp_submission,my_submission_zip_file) os.close(filehandle1) os.close(filehandle2) # cleanup shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp) #grade_items_logging.log_message(job_id,is_batch_job,"done zip",item_name) return (my_autograding_zip_file,my_submission_zip_file)
def grade_from_zip(my_autograding_zip_file, my_submission_zip_file, which_untrusted): os.chdir(SUBMITTY_DATA_DIR) tmp = os.path.join("/var/local/submitty/autograding_tmp/", which_untrusted, "tmp") # clean up old usage of this directory shutil.rmtree(tmp, ignore_errors=True) os.makedirs(tmp) which_machine = socket.gethostname() # unzip autograding and submission folders tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") unzip_this_file(my_autograding_zip_file, tmp_autograding) unzip_this_file(my_submission_zip_file, tmp_submission) os.remove(my_autograding_zip_file) os.remove(my_submission_zip_file) tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") queue_file = os.path.join(tmp_submission, "queue_file.json") with open(queue_file, 'r') as infile: queue_obj = json.load(infile) queue_time_longstring = queue_obj["queue_time"] waittime = queue_obj["waittime"] is_batch_job = queue_obj["regrade"] job_id = queue_obj["job_id"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" partial_path = os.path.join(queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"], queue_obj["course"], "submissions", partial_path) grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "wait:", waittime, "") # -------------------------------------------------------------------- # START DOCKER # WIP: This option file facilitated testing... #USE_DOCKER = os.path.isfile("/tmp/use_docker") #use_docker_string="grading begins, using DOCKER" if USE_DOCKER else "grading begins (not using docker)" #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message=use_docker_string) container = None if USE_DOCKER: container = subprocess.check_output([ 'docker', 'run', '-t', '-d', '-v', tmp + ':' + tmp, 'ubuntu:custom' ]).decode('utf8').strip() dockerlaunch_done = dateutils.get_current_time() dockerlaunch_time = (dockerlaunch_done - grading_began).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, submission_path, "dcct:", dockerlaunch_time, "docker container created") # -------------------------------------------------------------------- # COMPILE THE SUBMITTED CODE with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nCOMPILATION STARTS", file=f) # copy submitted files to the tmp compilation directory tmp_compilation = os.path.join(tmp, "TMP_COMPILATION") os.mkdir(tmp_compilation) os.chdir(tmp_compilation) submission_path = os.path.join(tmp_submission, "submission") checkout_path = os.path.join(tmp_submission, "checkout") provided_code_path = os.path.join(tmp_autograding, "provided_code") test_input_path = os.path.join(tmp_autograding, "test_input") test_output_path = os.path.join(tmp_autograding, "test_output") custom_validation_code_path = os.path.join(tmp_autograding, "custom_validation_code") bin_path = os.path.join(tmp_autograding, "bin") form_json_config = os.path.join(tmp_autograding, "form.json") complete_config = os.path.join(tmp_autograding, "complete_config.json") with open(form_json_config, 'r') as infile: gradeable_config_obj = json.load(infile) gradeable_deadline_string = gradeable_config_obj["date_due"] with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) patterns_submission_to_compilation = complete_config_obj["autograding"][ "submission_to_compilation"] pattern_copy("submission_to_compilation", patterns_submission_to_compilation, submission_path, tmp_compilation, tmp_logs) is_vcs = gradeable_config_obj["upload_type"] == "repository" checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) if is_vcs: pattern_copy("checkout_to_compilation", patterns_submission_to_compilation, checkout_subdir_path, tmp_compilation, tmp_logs) # copy any instructor provided code files to tmp compilation directory copy_contents_into(job_id, provided_code_path, tmp_compilation, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy compile.out to the current directory shutil.copy(os.path.join(bin_path, "compile.out"), os.path.join(tmp_compilation, "my_compile.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_compilation, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP) add_permissions(tmp, stat.S_IROTH | stat.S_IXOTH) add_permissions(tmp_logs, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() with open(os.path.join(tmp_logs, "compilation_log.txt"), 'w') as logfile: if USE_DOCKER: compile_success = subprocess.call([ 'docker', 'exec', '-w', tmp_compilation, container, os.path.join(tmp_compilation, 'my_compile.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string ], stdout=logfile) else: compile_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(tmp_compilation, "my_compile.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string ], stdout=logfile) if compile_success == 0: print(which_machine, which_untrusted, "COMPILATION OK") else: print(which_machine, which_untrusted, "COMPILATION FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="COMPILATION FAILURE") untrusted_grant_rwx_access(which_untrusted, tmp_compilation) # remove the compilation program os.remove(os.path.join(tmp_compilation, "my_compile.out")) # return to the main tmp directory os.chdir(tmp) # -------------------------------------------------------------------- # make the runner directory with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nRUNNER STARTS", file=f) tmp_work = os.path.join(tmp, "TMP_WORK") os.makedirs(tmp_work) os.chdir(tmp_work) # move all executable files from the compilation directory to the main tmp directory # Note: Must preserve the directory structure of compiled files (esp for Java) patterns_submission_to_runner = complete_config_obj["autograding"][ "submission_to_runner"] pattern_copy("submission_to_runner", patterns_submission_to_runner, submission_path, tmp_work, tmp_logs) if is_vcs: pattern_copy("checkout_to_runner", patterns_submission_to_runner, checkout_subdir_path, tmp_work, tmp_logs) patterns_compilation_to_runner = complete_config_obj["autograding"][ "compilation_to_runner"] pattern_copy("compilation_to_runner", patterns_compilation_to_runner, tmp_compilation, tmp_work, tmp_logs) # copy input files to tmp_work directory copy_contents_into(job_id, test_input_path, tmp_work, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy runner.out to the current directory shutil.copy(os.path.join(bin_path, "run.out"), os.path.join(tmp_work, "my_runner.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # run the run.out as the untrusted user with open(os.path.join(tmp_logs, "runner_log.txt"), 'w') as logfile: print("LOGGING BEGIN my_runner.out", file=logfile) logfile.flush() try: if USE_DOCKER: runner_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_runner.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string ], stdout=logfile) else: runner_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(tmp_work, "my_runner.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string ], stdout=logfile) logfile.flush() except Exception as e: print("ERROR caught runner.out exception={0}".format(str( e.args[0])).encode("utf-8"), file=logfile) logfile.flush() print("LOGGING END my_runner.out", file=logfile) logfile.flush() killall_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "killall.py") ], stdout=logfile) print("KILLALL COMPLETE my_runner.out", file=logfile) logfile.flush() if killall_success != 0: msg = 'RUNNER ERROR: had to kill {} process(es)'.format( killall_success) print("pid", os.getpid(), msg) grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "", "", msg) if runner_success == 0: print(which_machine, which_untrusted, "RUNNER OK") else: print(which_machine, which_untrusted, "RUNNER FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="RUNNER FAILURE") untrusted_grant_rwx_access(which_untrusted, tmp_work) untrusted_grant_rwx_access(which_untrusted, tmp_compilation) # -------------------------------------------------------------------- # RUN VALIDATOR with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nVALIDATION STARTS", file=f) # copy results files from compilation... patterns_submission_to_validation = complete_config_obj["autograding"][ "submission_to_validation"] pattern_copy("submission_to_validation", patterns_submission_to_validation, submission_path, tmp_work, tmp_logs) if is_vcs: pattern_copy("checkout_to_validation", patterns_submission_to_validation, checkout_subdir_path, tmp_work, tmp_logs) patterns_compilation_to_validation = complete_config_obj["autograding"][ "compilation_to_validation"] pattern_copy("compilation_to_validation", patterns_compilation_to_validation, tmp_compilation, tmp_work, tmp_logs) # remove the compilation directory shutil.rmtree(tmp_compilation) # copy output files to tmp_work directory copy_contents_into(job_id, test_output_path, tmp_work, tmp_logs) # copy any instructor custom validation code into the tmp work directory copy_contents_into(job_id, custom_validation_code_path, tmp_work, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy validator.out to the current directory shutil.copy(os.path.join(bin_path, "validate.out"), os.path.join(tmp_work, "my_validator.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(os.path.join(tmp_work, "my_validator.out"), stat.S_IROTH | stat.S_IXOTH) # validator the validator.out as the untrusted user with open(os.path.join(tmp_logs, "validator_log.txt"), 'w') as logfile: if USE_DOCKER: validator_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_validator.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string ], stdout=logfile) else: validator_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"), which_untrusted, os.path.join(tmp_work, "my_validator.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string ], stdout=logfile) if validator_success == 0: print(which_machine, which_untrusted, "VALIDATOR OK") else: print(which_machine, which_untrusted, "VALIDATOR FAILURE") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, message="VALIDATION FAILURE") untrusted_grant_rwx_access(which_untrusted, tmp_work) # grab the result of autograding grade_result = "" with open(os.path.join(tmp_work, "grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line # -------------------------------------------------------------------- # MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE tmp_results = os.path.join(tmp, "TMP_RESULTS") with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nARCHIVING STARTS", file=f) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) os.makedirs(os.path.join(tmp_results, "details")) patterns_work_to_details = complete_config_obj["autograding"][ "work_to_details"] pattern_copy("work_to_details", patterns_work_to_details, tmp_work, os.path.join(tmp_results, "details"), tmp_logs) history_file_tmp = os.path.join(tmp_submission, "history.json") history_file = os.path.join(tmp_results, "history.json") if os.path.isfile(history_file_tmp): shutil.move(history_file_tmp, history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file, int(HWCRON_UID), ta_group_id) add_permissions(history_file, stat.S_IRGRP) grading_finished = dateutils.get_current_time() shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results) # ------------------------------------------------------------- # create/append to the results history # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) gradeable_deadline_datetime = dateutils.read_submitty_date( gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date( gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int( (submission_datetime - gradeable_deadline_datetime).total_seconds()) # note: negative = not late with open(os.path.join(tmp_submission, ".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) grading_finished_longstring = dateutils.write_submitty_date( grading_finished) gradingtime = (grading_finished - grading_began).total_seconds() with open(os.path.join(tmp_submission, "queue_file.json"), 'r') as infile: queue_obj = json.load(infile) queue_obj["gradingtime"] = gradingtime queue_obj["grade_result"] = grade_result queue_obj["which_untrusted"] = which_untrusted with open(os.path.join(tmp_results, "queue_file.json"), 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) with open(os.path.join(tmp_work, "results.json"), 'r') as read_file: results_obj = json.load(read_file) if 'revision' in queue_obj.keys(): results_obj['revision'] = queue_obj['revision'] with open(os.path.join(tmp_results, "results.json"), 'w') as outfile: json.dump(results_obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) write_grade_history.just_write_grade_history( history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_time_longstring, is_batch_job_string, grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result) os.chdir(SUBMITTY_DATA_DIR) if USE_DOCKER: with open(os.path.join(tmp_logs, "overall_log.txt"), 'w') as logfile: chmod_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, 'chmod', '-R', 'o+rwx', '.' ], stdout=logfile) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: f.write("FINISHED GRADING!\n") # save the logs! shutil.copytree(tmp_logs, os.path.join(tmp_results, "logs")) # zip up results folder filehandle, my_results_zip_file = tempfile.mkstemp() zip_my_directory(tmp_results, my_results_zip_file) os.close(filehandle) shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp_results) shutil.rmtree(tmp_work) shutil.rmtree(tmp) # WIP: extra logging for testing #grade_items_logging.log_message(job_id,is_batch_job,which_untrusted,submission_path,message="done grading") # -------------------------------------------------------------------- # CLEAN UP DOCKER if USE_DOCKER: subprocess.call(['docker', 'rm', '-f', container]) dockerdestroy_done = dateutils.get_current_time() dockerdestroy_time = (dockerdestroy_done - grading_finished).total_seconds() grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, submission_path, "ddt:", dockerdestroy_time, "docker container destroyed") grade_items_logging.log_message(job_id, is_batch_job, which_untrusted, item_name, "grade:", gradingtime, grade_result) return my_results_zip_file