def shipper_process(my_name, which_machine,my_capabilities,which_untrusted,overall_lock): """ Each shipper process spins in a loop, looking for a job that matches the capabilities of this machine, and then oversees the autograding of that job. Interactive jobs are prioritized over batch (regrade) jobs. If no jobs are available, the shipper waits on an event editing one of the queues. """ # ignore keyboard interrupts in the shipper processes signal.signal(signal.SIGINT, signal.SIG_IGN) counter=0 while True: try: my_job = get_job(my_name,which_machine,my_capabilities,which_untrusted,overall_lock) if not my_job == "": counter=0 grade_queue_file(my_name,which_machine,which_untrusted,os.path.join(INTERACTIVE_QUEUE,my_job)) continue else: if counter == 0 or counter >= 10: print (my_name,which_untrusted,"no available job") counter=0 counter+=1 time.sleep(1) except Exception as e: my_message = "ERROR in get_job " + which_machine + " " + which_untrusted + " " + str(e) print (my_message) grade_items_logging.log_message(JOB_ID, message=my_message) time.sleep(1)
def copy_contents_into(source,target,tmp_logs): if not os.path.isdir(target): grade_items_logging.log_message("ERROR: the target directory does not exist " + target) raise SystemExit("ERROR: the target directory does not exist '", target, "'") if os.path.isdir(source): for item in os.listdir(source): if os.path.isdir(os.path.join(source,item)): if os.path.isdir(os.path.join(target,item)): # recurse copy_contents_into(os.path.join(source,item),os.path.join(target,item),tmp_logs) elif os.path.isfile(os.path.join(target,item)): grade_items_logging.log_message("ERROR: the target subpath is a file not a directory '" + os.path.join(target,item) + "'") raise SystemExit("ERROR: the target subpath is a file not a directory '", os.path.join(target,item), "'") else: # copy entire subtree shutil.copytree(os.path.join(source,item),os.path.join(target,item)) else: if os.path.exists(os.path.join(target,item)): with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("\nWARNING: REMOVING DESTINATION FILE" , os.path.join(target,item), " THEN OVERWRITING: ", os.path.join(source,item), "\n", file=f) os.remove(os.path.join(target,item)) try: shutil.copy(os.path.join(source,item),target) except: raise SystemExit("ERROR COPYING FILE: " + os.path.join(source,item) + " -> " + os.path.join(target,item))
def get_submission_path(next_directory,next_to_grade): queue_file = os.path.join(next_directory,next_to_grade) if not os.path.isfile(queue_file): grade_items_logging.log_message("ERROR: the file does not exist " + queue_file) raise SystemExit("ERROR: the file does not exist",queue_file) with open(queue_file, 'r') as infile: obj = json.load(infile) return obj
def load_queue_file_obj(next_directory, next_to_grade): queue_file = os.path.join(next_directory, next_to_grade) if not os.path.isfile(queue_file): grade_items_logging.log_message( message="ERROR: the file does not exist " + queue_file) raise RuntimeError("ERROR: the file does not exist", queue_file) with open(queue_file, 'r') as infile: obj = json.load(infile) return obj
def populate_queue(queue, folder): """ Populate a queue with all files in folder. We first scan the folder to check for any "GRADING_*" and clean them up, and then add the remaining files to the queue, sorted by creation time. :param queue: multiprocessing.queues.Queue :param folder: string representing the path to the folder to add files from """ for file_path in glob.glob(os.path.join(folder, "GRADING_*")): grade_items_logging.log_message(message="Remove old queue file: " + file_path) os.remove(file_path) # Grab all the files currently in the folder, sorted by creation # time, and put them in the queue to be graded files = glob.glob(os.path.join(folder, "*")) files.sort(key=os.path.getctime) for f in files: queue.put(os.path.join(folder, f))
def unpack_grading_results_zip(which_machine, which_untrusted, my_results_zip_file): os.chdir(SUBMITTY_DATA_DIR) queue_obj = unzip_queue_file(my_results_zip_file) job_id = queue_obj["job_id"] partial_path = os.path.join(queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"], queue_obj["course"], "submissions", partial_path) results_path = os.path.join(SUBMITTY_DATA_DIR, "courses", queue_obj["semester"], queue_obj["course"], "results", partial_path) # clean out all of the old files if this is a re-run shutil.rmtree(results_path, ignore_errors=True) # create the directory (and the full path if it doesn't already exist) os.makedirs(results_path) # unzip the file & clean up unzip_this_file(my_results_zip_file, results_path) os.remove(my_results_zip_file) # add information to the database insert_database_version_data.insert_to_database( queue_obj["semester"], queue_obj["course"], queue_obj["gradeable"], queue_obj["user"], queue_obj["team"], queue_obj["who"], True if queue_obj["is_team"] else False, str(queue_obj["version"])) submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name) is_batch_job = queue_obj["regrade"] gradingtime = queue_obj["gradingtime"] grade_result = queue_obj["grade_result"] print(which_machine, which_untrusted, "unzip", item_name, " in ", int(gradingtime), " seconds") grade_items_logging.log_message(job_id, is_batch_job, "unzip", item_name, "grade:", gradingtime, grade_result)
def grade_queue_file(queue_file,which_untrusted): """ Grades a single item in one of the queues. :param queue_file: file path pointing to the file we want to operate one. """ my_dir,my_file=os.path.split(queue_file) pid = os.getpid() directory = os.path.dirname(os.path.realpath(queue_file)) name = os.path.basename(os.path.realpath(queue_file)) grading_file = os.path.join(directory, "GRADING_" + name) open(os.path.join(grading_file), "w").close() #untrusted = multiprocessing.current_process().untrusted try: grade_item.just_grade_item(my_dir, queue_file, which_untrusted) except Exception as e: print ("ERROR attempting to grade item: ", queue_file, " exception=",e) grade_items_logging.log_message(message="ERROR attempting to grade item: " + queue_file + " exception " + repr(e)) # note: not necessary to acquire lock for these statements, but # make sure you remove the queue file, then the grading file try: os.remove(queue_file) except: print ("ERROR attempting to remove queue file: ", queue_file) grade_items_logging.log_message(message="ERROR attempting to remove queue file: " + queue_file) try: os.remove(grading_file) except: print ("ERROR attempting to remove grading file: ", grading_file) grade_items_logging.log_message(message="ERROR attempting to remove grading file: " + grading_file)
def just_grade_item(next_directory, next_to_grade, which_untrusted): # verify the hwcron user is running this script if not int(os.getuid()) == int(HWCRON_UID): grade_items_logging.log_message(message="ERROR: must be run by hwcron") raise SystemExit( "ERROR: the grade_item.py script must be run by the hwcron user") # prepare the zip files try: autograding_zip, submission_zip = prepare_autograding_and_submission_zip( next_directory, next_to_grade) except: grade_items_logging.log_message( jobname=next_to_grade, message= "ERROR: Exception when preparing autograding and submission zip") return # actually do the grading (this step could be shipped to another machine) try: results_zip = grade_from_zip(autograding_zip, submission_zip, which_untrusted) except: grade_items_logging.log_message( jobname=next_to_grade, message="ERROR: Exception when grading from zip") with contextlib.suppress(FileNotFoundError): os.remove(autograding_zip) with contextlib.suppress(FileNotFoundError): os.remove(submission_zip) return # archive the results of grading try: unpack_grading_results_zip(results_zip) except: grade_items_logging.log_message( jobname=next_to_grade, message="ERROR: Exception when unpacking zip") with contextlib.suppress(FileNotFoundError): os.remove(results_zip) return
def grade_queue_file(my_name, which_machine,which_untrusted,queue_file): """ Oversees the autograding of single item from the queue :param queue_file: details of what to grade :param which_machine: name of machine to send this job to (might be "localhost") :param which_untrusted: specific untrusted user for this autograding job """ my_dir,my_file=os.path.split(queue_file) pid = os.getpid() directory = os.path.dirname(os.path.realpath(queue_file)) name = os.path.basename(os.path.realpath(queue_file)) grading_file = os.path.join(directory, "GRADING_" + name) #TODO: breach which_machine into id, address, and passphrase. try: # prepare the job shipper_counter=0 while not prepare_job(my_name,which_machine, which_untrusted, my_dir, queue_file): shipper_counter = 0 time.sleep(1) if shipper_counter >= 10: prints(my_name, which_untrusted, "shipper prep loop: ",queue_file) shipper_counter=0 # then wait for grading to be completed shipper_counter=0 while not unpack_job(which_machine, which_untrusted, my_dir, queue_file): shipper_counter+=1 time.sleep(1) if shipper_counter >= 10: print (my_name,which_untrusted,"shipper wait for grade: ",queue_file) shipper_counter=0 except Exception as e: print (my_name, " ERROR attempting to grade item: ", queue_file, " exception=",str(e)) grade_items_logging.log_message(JOB_ID, message=str(my_name)+" ERROR attempting to grade item: " + queue_file + " exception " + repr(e)) # note: not necessary to acquire lock for these statements, but # make sure you remove the queue file, then the grading file try: os.remove(queue_file) except Exception as e: print (my_name, " ERROR attempting to remove queue file: ", queue_file, " exception=",str(e)) grade_items_logging.log_message(JOB_ID, message=str(my_name)+" ERROR attempting to remove queue file: " + queue_file + " exception=" + str(e)) try: os.remove(grading_file) except Exception as e: print (my_name, " ERROR attempting to remove grading file: ", grading_file, " exception=",str(e)) grade_items_logging.log_message(JOB_ID, message=str(my_name)+" ERROR attempting to remove grading file: " + grading_file + " exception=" + str(e))
def get_job(my_name,which_machine,my_capabilities,which_untrusted,overall_lock): """ Picks a job from the queue :param overall_lock: a lock on the directory containing all queue files """ time_get_job_begin = dateutils.get_current_time() overall_lock.acquire() folder= INTERACTIVE_QUEUE # Grab all the files currently in the folder, sorted by creation # time, and put them in the queue to be graded files = glob.glob(os.path.join(folder, "*")) files_and_times = list() for f in files: try: my_time = os.path.getctime(f) except: continue tup = (f, my_time) files_and_times.append(tup) files_and_times = sorted(files_and_times, key=operator.itemgetter(1)) my_job="" for full_path_file, file_time in files_and_times: # get the file name (without the path) just_file = full_path_file[len(folder)+1:] # skip items that are already being graded if (just_file[0:8]=="GRADING_"): continue grading_file = os.path.join(folder,"GRADING_"+just_file) if grading_file in files: continue # found something to do try: with open(full_path_file, 'r') as infile: queue_obj = json.load(infile) except: continue #Check to make sure that we are capable of grading this submission required_capabilities = queue_obj["required_capabilities"] if not required_capabilities in my_capabilities: continue # prioritize interactive jobs over (batch) regrades # if you've found an interactive job, exit early (since they are sorted by timestamp) if not "regrade" in queue_obj or not queue_obj["regrade"]: my_job = just_file break # otherwise it's a regrade, and if we don't already have a # job, take it, but we have to search the rest of the list if my_job == "": my_job = just_file if not my_job == "": grading_file = os.path.join(folder, "GRADING_" + my_job) # create the grading file open(os.path.join(grading_file), "w").close() overall_lock.release() time_get_job_end = dateutils.get_current_time() time_delta = time_get_job_end-time_get_job_begin if time_delta > datetime.timedelta(milliseconds=100): print (my_name, " WARNING: submitty_autograding shipper get_job time ", time_delta) grade_items_logging.log_message(JOB_ID, message=str(my_name)+" WARNING: submitty_autograding shipper get_job time "+str(time_delta)) return my_job
def grade_from_zip(my_autograding_zip_file, my_submission_zip_file, which_untrusted): os.chdir(SUBMITTY_DATA_DIR) tmp = os.path.join("/var/local/submitty/autograding_tmp/", which_untrusted, "tmp") # clean up old usage of this directory shutil.rmtree(tmp, ignore_errors=True) os.makedirs(tmp) # unzip autograding and submission folders tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") unzip_this_file(my_autograding_zip_file, tmp_autograding) unzip_this_file(my_submission_zip_file, tmp_submission) os.remove(my_autograding_zip_file) os.remove(my_submission_zip_file) tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") queue_file = os.path.join(tmp_submission, "queue_file.json") with open(queue_file, 'r') as infile: queue_obj = json.load(infile) queue_time_longstring = queue_obj["queue_time"] waittime = queue_obj["waittime"] is_batch_job = queue_obj["is_batch_job"] is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" partial_path = os.path.join(queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"])) item_name = os.path.join(queue_obj["semester"], queue_obj["course"], "submissions", partial_path) grade_items_logging.log_message(is_batch_job, which_untrusted, item_name, "wait:", waittime, "") # -------------------------------------------------------------------- # START DOCKER # WIP: This option file facilitated testing... #USE_DOCKER = os.path.isfile("/tmp/use_docker") #use_docker_string="grading begins, using DOCKER" if USE_DOCKER else "grading begins (not using docker)" #grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,message=use_docker_string) container = None if USE_DOCKER: container = subprocess.check_output([ 'docker', 'run', '-t', '-d', '-v', tmp + ':' + tmp, 'ubuntu:custom' ]).decode('utf8').strip() dockerlaunch_done = dateutils.get_current_time() dockerlaunch_time = (dockerlaunch_done - grading_began).total_seconds() grade_items_logging.log_message(is_batch_job, which_untrusted, submission_path, "dcct:", dockerlaunch_time, "docker container created") # -------------------------------------------------------------------- # COMPILE THE SUBMITTED CODE with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nCOMPILATION STARTS", file=f) # copy submitted files to the tmp compilation directory tmp_compilation = os.path.join(tmp, "TMP_COMPILATION") os.mkdir(tmp_compilation) os.chdir(tmp_compilation) submission_path = os.path.join(tmp_submission, "submission") checkout_path = os.path.join(tmp_submission, "checkout") provided_code_path = os.path.join(tmp_autograding, "provided_code") test_input_path = os.path.join(tmp_autograding, "test_input") test_output_path = os.path.join(tmp_autograding, "test_output") custom_validation_code_path = os.path.join(tmp_autograding, "custom_validation_code") bin_path = os.path.join(tmp_autograding, "bin") form_json_config = os.path.join(tmp_autograding, "form.json") complete_config = os.path.join(tmp_autograding, "complete_config.json") with open(form_json_config, 'r') as infile: gradeable_config_obj = json.load(infile) gradeable_deadline_string = gradeable_config_obj["date_due"] with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) patterns_submission_to_compilation = complete_config_obj["autograding"][ "submission_to_compilation"] pattern_copy("submission_to_compilation", patterns_submission_to_compilation, submission_path, tmp_compilation, tmp_logs) is_vcs = gradeable_config_obj["upload_type"] == "repository" checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) if is_vcs: pattern_copy("checkout_to_compilation", patterns_submission_to_compilation, checkout_subdir_path, tmp_compilation, tmp_logs) # copy any instructor provided code files to tmp compilation directory copy_contents_into(provided_code_path, tmp_compilation, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy compile.out to the current directory shutil.copy(os.path.join(bin_path, "compile.out"), os.path.join(tmp_compilation, "my_compile.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_compilation, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP) add_permissions(tmp, stat.S_IROTH | stat.S_IXOTH) add_permissions(tmp_logs, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp"), 'r') as submission_time_file: submission_string = submission_time_file.read().rstrip() with open(os.path.join(tmp_logs, "compilation_log.txt"), 'w') as logfile: if USE_DOCKER: compile_success = subprocess.call([ 'docker', 'exec', '-w', tmp_compilation, container, os.path.join(tmp_compilation, 'my_compile.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string ], stdout=logfile) else: compile_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "bin", "untrusted_execute"), which_untrusted, os.path.join(tmp_compilation, "my_compile.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string ], stdout=logfile) if compile_success == 0: print("pid", os.getpid(), "COMPILATION OK") else: print("pid", os.getpid(), "COMPILATION FAILURE") grade_items_logging.log_message(is_batch_job, which_untrusted, item_name, message="COMPILATION FAILURE") untrusted_grant_rwx_access(which_untrusted, tmp_compilation) # remove the compilation program os.remove(os.path.join(tmp_compilation, "my_compile.out")) # return to the main tmp directory os.chdir(tmp) # -------------------------------------------------------------------- # make the runner directory with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nRUNNER STARTS", file=f) tmp_work = os.path.join(tmp, "TMP_WORK") os.makedirs(tmp_work) os.chdir(tmp_work) # move all executable files from the compilation directory to the main tmp directory # Note: Must preserve the directory structure of compiled files (esp for Java) patterns_submission_to_runner = complete_config_obj["autograding"][ "submission_to_runner"] pattern_copy("submission_to_runner", patterns_submission_to_runner, submission_path, tmp_work, tmp_logs) if is_vcs: pattern_copy("checkout_to_runner", patterns_submission_to_runner, checkout_subdir_path, tmp_work, tmp_logs) patterns_compilation_to_runner = complete_config_obj["autograding"][ "compilation_to_runner"] pattern_copy("compilation_to_runner", patterns_compilation_to_runner, tmp_compilation, tmp_work, tmp_logs) # copy input files to tmp_work directory copy_contents_into(test_input_path, tmp_work, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy runner.out to the current directory shutil.copy(os.path.join(bin_path, "run.out"), os.path.join(tmp_work, "my_runner.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # run the run.out as the untrusted user with open(os.path.join(tmp_logs, "runner_log.txt"), 'w') as logfile: print("LOGGING BEGIN my_runner.out", file=logfile) logfile.flush() try: if USE_DOCKER: runner_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_runner.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string ], stdout=logfile) else: runner_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "bin", "untrusted_execute"), which_untrusted, os.path.join(tmp_work, "my_runner.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string ], stdout=logfile) logfile.flush() except Exception as e: print("ERROR caught runner.out exception={0}".format(str( e.args[0])).encode("utf-8"), file=logfile) logfile.flush() print("LOGGING END my_runner.out", file=logfile) logfile.flush() killall_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "bin", "untrusted_execute"), which_untrusted, os.path.join(SUBMITTY_INSTALL_DIR, "bin", "killall.py") ], stdout=logfile) print("KILLALL COMPLETE my_runner.out", file=logfile) logfile.flush() if killall_success != 0: msg = 'RUNNER ERROR: had to kill {} process(es)'.format( killall_success) print("pid", os.getpid(), msg) grade_items_logging.log_message(is_batch_job, which_untrusted, item_name, "", "", msg) if runner_success == 0: print("pid", os.getpid(), "RUNNER OK") else: print("pid", os.getpid(), "RUNNER FAILURE") grade_items_logging.log_message(is_batch_job, which_untrusted, item_name, message="RUNNER FAILURE") untrusted_grant_rwx_access(which_untrusted, tmp_work) untrusted_grant_rwx_access(which_untrusted, tmp_compilation) # -------------------------------------------------------------------- # RUN VALIDATOR with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nVALIDATION STARTS", file=f) # copy results files from compilation... patterns_submission_to_validation = complete_config_obj["autograding"][ "submission_to_validation"] pattern_copy("submission_to_validation", patterns_submission_to_validation, submission_path, tmp_work, tmp_logs) if is_vcs: pattern_copy("checkout_to_validation", patterns_submission_to_validation, checkout_subdir_path, tmp_work, tmp_logs) patterns_compilation_to_validation = complete_config_obj["autograding"][ "compilation_to_validation"] pattern_copy("compilation_to_validation", patterns_compilation_to_validation, tmp_compilation, tmp_work, tmp_logs) # remove the compilation directory shutil.rmtree(tmp_compilation) # copy output files to tmp_work directory copy_contents_into(test_output_path, tmp_work, tmp_logs) # copy any instructor custom validation code into the tmp work directory copy_contents_into(custom_validation_code_path, tmp_work, tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy validator.out to the current directory shutil.copy(os.path.join(bin_path, "validate.out"), os.path.join(tmp_work, "my_validator.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(os.path.join(tmp_work, "my_validator.out"), stat.S_IROTH | stat.S_IXOTH) # validator the validator.out as the untrusted user with open(os.path.join(tmp_logs, "validator_log.txt"), 'w') as logfile: if USE_DOCKER: validator_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_validator.out'), queue_obj['gradeable'], queue_obj['who'], str(queue_obj['version']), submission_string ], stdout=logfile) else: validator_success = subprocess.call([ os.path.join(SUBMITTY_INSTALL_DIR, "bin", "untrusted_execute"), which_untrusted, os.path.join(tmp_work, "my_validator.out"), queue_obj["gradeable"], queue_obj["who"], str(queue_obj["version"]), submission_string ], stdout=logfile) if validator_success == 0: print("pid", os.getpid(), "VALIDATOR OK") else: print("pid", os.getpid(), "VALIDATOR FAILURE") grade_items_logging.log_message(is_batch_job, which_untrusted, item_name, message="VALIDATION FAILURE") untrusted_grant_rwx_access(which_untrusted, tmp_work) # grab the result of autograding grade_result = "" with open(os.path.join(tmp_work, "grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line # -------------------------------------------------------------------- # MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE tmp_results = os.path.join(tmp, "TMP_RESULTS") with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nARCHIVING STARTS", file=f) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) os.makedirs(os.path.join(tmp_results, "details")) patterns_work_to_details = complete_config_obj["autograding"][ "work_to_details"] pattern_copy("work_to_details", patterns_work_to_details, tmp_work, os.path.join(tmp_results, "details"), tmp_logs) history_file_tmp = os.path.join(tmp_submission, "history.json") history_file = os.path.join(tmp_results, "history.json") if os.path.isfile(history_file_tmp): shutil.move(history_file_tmp, history_file) # fix permissions ta_group_id = os.stat(tmp_results).st_gid os.chown(history_file, int(HWCRON_UID), ta_group_id) add_permissions(history_file, stat.S_IRGRP) grading_finished = dateutils.get_current_time() shutil.copy(os.path.join(tmp_work, "results.json"), tmp_results) shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results) # ------------------------------------------------------------- # create/append to the results history # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) gradeable_deadline_datetime = dateutils.read_submitty_date( gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date( gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int( (submission_datetime - gradeable_deadline_datetime).total_seconds()) # note: negative = not late with open(os.path.join(tmp_submission, ".grading_began"), 'r') as f: grading_began_longstring = f.read() grading_began = dateutils.read_submitty_date(grading_began_longstring) grading_finished_longstring = dateutils.write_submitty_date( grading_finished) gradingtime = (grading_finished - grading_began).total_seconds() with open(os.path.join(tmp_submission, "queue_file.json"), 'r') as infile: queue_obj = json.load(infile) queue_obj["gradingtime"] = gradingtime queue_obj["grade_result"] = grade_result queue_obj["which_untrusted"] = which_untrusted with open(os.path.join(tmp_results, "queue_file.json"), 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) write_grade_history.just_write_grade_history( history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_time_longstring, is_batch_job_string, grading_began_longstring, int(waittime), grading_finished_longstring, int(gradingtime), grade_result) os.chdir(SUBMITTY_DATA_DIR) if USE_DOCKER: with open(os.path.join(tmp_logs, "overall_log.txt"), 'w') as logfile: chmod_success = subprocess.call([ 'docker', 'exec', '-w', tmp_work, container, 'chmod', '-R', 'o+rwx', '.' ], stdout=logfile) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: f.write("FINISHED GRADING!\n") # save the logs! shutil.copytree(tmp_logs, os.path.join(tmp_results, "logs")) # zip up results folder my_results_zip_file = tempfile.mkstemp()[1] zip_my_directory(tmp_results, my_results_zip_file) shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp_results) shutil.rmtree(tmp_work) shutil.rmtree(tmp) # WIP: extra logging for testing #grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,message="done grading") # -------------------------------------------------------------------- # CLEAN UP DOCKER if USE_DOCKER: subprocess.call(['docker', 'rm', '-f', container]) dockerdestroy_done = dateutils.get_current_time() dockerdestroy_time = (dockerdestroy_done - grading_finished).total_seconds() grade_items_logging.log_message(is_batch_job, which_untrusted, submission_path, "ddt:", dockerdestroy_time, "docker container destroyed") grade_items_logging.log_message(is_batch_job, which_untrusted, item_name, "grade:", gradingtime, grade_result) return my_results_zip_file
def just_grade_item(next_directory,next_to_grade,which_untrusted): my_pid = os.getpid() # verify the hwcron user is running this script if not int(os.getuid()) == int(HWCRON_UID): grade_items_logging.log_message("ERROR: must be run by hwcron") raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user") # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = get_submission_path(next_directory,next_to_grade) submission_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"], "submissions",obj["gradeable"],obj["who"],str(obj["version"])) if not os.path.isdir(submission_path): grade_items_logging.log_message("ERROR: the submission directory does not exist" + submission_path) raise SystemExit("ERROR: the submission directory does not exist",submission_path) print ("pid",my_pid,"GRADE THIS", submission_path) is_vcs,vcs_type,vcs_base_url,vcs_subdirectory = get_vcs_info(SUBMITTY_DATA_DIR,obj["semester"],obj["course"],obj["gradeable"],obj["who"],obj["team"]) is_batch_job = next_directory==BATCH_QUEUE is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory,next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = int((grading_began-queue_time).total_seconds()) grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"wait:",waittime,"") # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"provided_code",obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_input",obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_output",obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"custom_validation_code",obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"bin") checkout_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"checkout",obj["gradeable"],obj["who"],str(obj["version"])) results_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"results",obj["gradeable"],obj["who"],str(obj["version"])) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path,"history.json") history_file_tmp = "" if os.path.isfile(history_file): filehandle,history_file_tmp = tempfile.mkstemp() shutil.copy(history_file,history_file_tmp) # get info from the gradeable config file json_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","form","form_"+obj["gradeable"]+".json") with open(json_config, 'r') as infile: gradeable_config_obj = json.load(infile) # get info from the gradeable config file complete_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","complete_config","complete_config_"+obj["gradeable"]+".json") with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get("use_checkout_subdirectory","") checkout_subdir_path = os.path.join(checkout_path,checkout_subdirectory) # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = os.path.join("/var/local/submitty/autograding_tmp/",which_untrusted,"tmp") shutil.rmtree(tmp,ignore_errors=True) os.makedirs(tmp) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp,"tmp_logs") os.makedirs(tmp_logs) # grab the submission time with open (os.path.join(submission_path,".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) # -------------------------------------------------------------------- # CHECKOUT THE STUDENT's REPO if is_vcs: with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nVCS CHECKOUT", file=f) print ("vcs_subdirectory",vcs_subdirectory, file=f) # cleanup the previous checkout (if it exists) shutil.rmtree(checkout_path,ignore_errors=True) os.makedirs(checkout_path, exist_ok=True) subprocess.call (['/usr/bin/git', 'clone', vcs_subdirectory, checkout_path]) os.chdir(checkout_path) # determine which version we need to checkout what_version = subprocess.check_output(['git', 'rev-list', '-n', '1', '--before="'+submission_string+'"', 'master']) what_version = str(what_version.decode('utf-8')).rstrip() if what_version == "": # oops, pressed the grade button before a valid commit shutil.rmtree(checkout_path,ignore_errors=True) else: # and check out the right version subprocess.call (['git', 'checkout', '-b', 'grade', what_version]) os.chdir(tmp) subprocess.call(['ls', '-lR', checkout_path], stdout=open(tmp_logs + "/overall.txt", 'a')) # -------------------------------------------------------------------- # START DOCKER container = None if USE_DOCKER: container = subprocess.check_output(['docker', 'run', '-t', '-d', '-v', tmp + ':' + tmp, 'ubuntu:custom']).decode('utf8').strip() # -------------------------------------------------------------------- # COMPILE THE SUBMITTED CODE with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nCOMPILATION STARTS", file=f) # copy submitted files to the tmp compilation directory tmp_compilation = os.path.join(tmp,"TMP_COMPILATION") os.mkdir(tmp_compilation) os.chdir(tmp_compilation) gradeable_deadline_string = gradeable_config_obj["date_due"] patterns_submission_to_compilation = complete_config_obj["autograding"]["submission_to_compilation"] pattern_copy("submission_to_compilation",patterns_submission_to_compilation,submission_path,tmp_compilation,tmp_logs) if is_vcs: pattern_copy("checkout_to_compilation",patterns_submission_to_compilation,checkout_subdir_path,tmp_compilation,tmp_logs) # copy any instructor provided code files to tmp compilation directory copy_contents_into(provided_code_path,tmp_compilation,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy compile.out to the current directory shutil.copy (os.path.join(bin_path,obj["gradeable"],"compile.out"),os.path.join(tmp_compilation,"my_compile.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_compilation, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP, stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP) add_permissions(tmp,stat.S_IROTH | stat.S_IXOTH) add_permissions(tmp_logs,stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) with open(os.path.join(tmp_logs,"compilation_log.txt"), 'w') as logfile: if USE_DOCKER: compile_success = subprocess.call(['docker', 'exec', '-w', tmp_compilation, container, os.path.join(tmp_compilation, 'my_compile.out'), obj['gradeable'], obj['who'], str(obj['version']), submission_string], stdout=logfile) else: compile_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"), which_untrusted, os.path.join(tmp_compilation,"my_compile.out"), obj["gradeable"], obj["who"], str(obj["version"]), submission_string], stdout=logfile) if compile_success == 0: print ("pid",my_pid,"COMPILATION OK") else: print ("pid",my_pid,"COMPILATION FAILURE") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","COMPILATION FAILURE") #raise SystemExit() untrusted_grant_rwx_access(which_untrusted,tmp_compilation) # remove the compilation program os.remove(os.path.join(tmp_compilation,"my_compile.out")) # return to the main tmp directory os.chdir(tmp) # -------------------------------------------------------------------- # make the runner directory with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nRUNNER STARTS", file=f) tmp_work = os.path.join(tmp,"TMP_WORK") os.makedirs(tmp_work) os.chdir(tmp_work) # move all executable files from the compilation directory to the main tmp directory # Note: Must preserve the directory structure of compiled files (esp for Java) patterns_submission_to_runner = complete_config_obj["autograding"]["submission_to_runner"] pattern_copy("submission_to_runner",patterns_submission_to_runner,submission_path,tmp_work,tmp_logs) if is_vcs: pattern_copy("checkout_to_runner",patterns_submission_to_runner,checkout_subdir_path,tmp_work,tmp_logs) patterns_compilation_to_runner = complete_config_obj["autograding"]["compilation_to_runner"] pattern_copy("compilation_to_runner",patterns_compilation_to_runner,tmp_compilation,tmp_work,tmp_logs) # copy input files to tmp_work directory copy_contents_into(test_input_path,tmp_work,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy runner.out to the current directory shutil.copy (os.path.join(bin_path,obj["gradeable"],"run.out"),os.path.join(tmp_work,"my_runner.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # raise SystemExit() # run the run.out as the untrusted user with open(os.path.join(tmp_logs,"runner_log.txt"), 'w') as logfile: if USE_DOCKER: runner_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_runner.out'), obj['gradeable'], obj['who'], str(obj['version']), submission_string], stdout=logfile) else: runner_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"), which_untrusted, os.path.join(tmp_work,"my_runner.out"), obj["gradeable"], obj["who"], str(obj["version"]), submission_string], stdout=logfile) if runner_success == 0: print ("pid",my_pid,"RUNNER OK") else: print ("pid",my_pid,"RUNNER FAILURE") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","RUNNER FAILURE") untrusted_grant_rwx_access(which_untrusted,tmp_work) untrusted_grant_rwx_access(which_untrusted,tmp_compilation) # -------------------------------------------------------------------- # RUN VALIDATOR with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nVALIDATION STARTS", file=f) # copy results files from compilation... patterns_submission_to_validation = complete_config_obj["autograding"]["submission_to_validation"] pattern_copy("submission_to_validation",patterns_submission_to_validation,submission_path,tmp_work,tmp_logs) if is_vcs: pattern_copy("checkout_to_validation",patterns_submission_to_validation,checkout_subdir_path,tmp_work,tmp_logs) patterns_compilation_to_validation = complete_config_obj["autograding"]["compilation_to_validation"] pattern_copy("compilation_to_validation",patterns_compilation_to_validation,tmp_compilation,tmp_work,tmp_logs) # remove the compilation directory shutil.rmtree(tmp_compilation) # copy output files to tmp_work directory copy_contents_into(test_output_path,tmp_work,tmp_logs) # copy any instructor custom validation code into the tmp work directory copy_contents_into(custom_validation_code_path,tmp_work,tmp_logs) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) # copy validator.out to the current directory shutil.copy (os.path.join(bin_path,obj["gradeable"],"validate.out"),os.path.join(tmp_work,"my_validator.out")) # give the untrusted user read/write/execute permissions on the tmp directory & files add_permissions_recursive(tmp_work, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) add_permissions(os.path.join(tmp_work,"my_validator.out"),stat.S_IROTH | stat.S_IXOTH) # validator the validator.out as the untrusted user with open(os.path.join(tmp_logs,"validator_log.txt"), 'w') as logfile: if USE_DOCKER: validator_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container, os.path.join(tmp_work, 'my_validator.out'), obj['gradeable'], obj['who'], str(obj['version']), submission_string], stdout=logfile) else: validator_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"), which_untrusted, os.path.join(tmp_work,"my_validator.out"), obj["gradeable"], obj["who"], str(obj["version"]), submission_string], stdout=logfile) if validator_success == 0: print ("pid",my_pid,"VALIDATOR OK") else: print ("pid",my_pid,"VALIDATOR FAILURE") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","VALIDATION FAILURE") untrusted_grant_rwx_access(which_untrusted,tmp_work) # grab the result of autograding grade_result = "" with open(os.path.join(tmp_work,"grade.txt")) as f: lines = f.readlines() for line in lines: line = line.rstrip('\n') if line.startswith("Automatic grading total:"): grade_result = line # -------------------------------------------------------------------- # MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: print ("====================================\nARCHIVING STARTS", file=f) subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a')) os.chdir(bin_path) # save the old results path! if os.path.isdir(os.path.join(results_path,"OLD")): shutil.move(os.path.join(results_path,"OLD"), os.path.join(tmp,"OLD_RESULTS")) # clean out all of the old files if this is a re-run shutil.rmtree(results_path,ignore_errors=True) # create the directory (and the full path if it doesn't already exist) os.makedirs(results_path) # bring back the old results! if os.path.isdir(os.path.join(tmp,"OLD_RESULTS")): shutil.move(os.path.join(tmp,"OLD_RESULTS"), os.path.join(results_path,"OLD")) os.makedirs(os.path.join(results_path,"details")) patterns_work_to_details = complete_config_obj["autograding"]["work_to_details"] pattern_copy("work_to_details",patterns_work_to_details,tmp_work,os.path.join(results_path,"details"),tmp_logs) if not history_file_tmp == "": shutil.move(history_file_tmp,history_file) # fix permissions ta_group_id = os.stat(results_path).st_gid os.chown(history_file,int(HWCRON_UID),ta_group_id) add_permissions(history_file,stat.S_IRGRP) grading_finished = dateutils.get_current_time() shutil.copy(os.path.join(tmp_work,"results.json"),results_path) shutil.copy(os.path.join(tmp_work,"grade.txt"),results_path) # ------------------------------------------------------------- # create/append to the results history gradeable_deadline_datetime = dateutils.read_submitty_date(gradeable_deadline_string) gradeable_deadline_longstring = dateutils.write_submitty_date(gradeable_deadline_datetime) submission_longstring = dateutils.write_submitty_date(submission_datetime) seconds_late = int((submission_datetime-gradeable_deadline_datetime).total_seconds()) # note: negative = not late grading_began_longstring = dateutils.write_submitty_date(grading_began) grading_finished_longstring = dateutils.write_submitty_date(grading_finished) gradingtime = int((grading_finished-grading_began).total_seconds()) write_grade_history.just_write_grade_history(history_file, gradeable_deadline_longstring, submission_longstring, seconds_late, queue_time_longstring, is_batch_job_string, grading_began_longstring, waittime, grading_finished_longstring, gradingtime, grade_result) #--------------------------------------------------------------------- # WRITE OUT VERSION DETAILS if WRITE_DATABASE: insert_database_version_data.insert_to_database( obj["semester"], obj["course"], obj["gradeable"], obj["user"], obj["team"], obj["who"], True if obj["is_team"] else False, str(obj["version"])) print ("pid",my_pid,"finished grading ", next_to_grade, " in ", gradingtime, " seconds") grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"grade:",gradingtime,grade_result) with open(os.path.join(tmp_logs,"overall.txt"),'a') as f: f.write("FINISHED GRADING!") # save the logs! shutil.copytree(tmp_logs,os.path.join(results_path,"logs")) # -------------------------------------------------------------------- # REMOVE TEMP DIRECTORY shutil.rmtree(tmp) # -------------------------------------------------------------------- # CLEAN UP DOCKER if USE_DOCKER: subprocess.call(['docker', 'rm', '-f', container])
def launch_workers(num_workers): # verify the hwcron user is running this script if not int(os.getuid()) == int(HWCRON_UID): raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user") grade_items_logging.log_message(message="grade_scheduler.py launched") # prepare a list of untrusted users to be used by the workers untrusted_users = multiprocessing.Queue() for i in range(num_workers): untrusted_users.put("untrusted" + str(i).zfill(2)) # Set up our queues that we're going to monitor for new jobs to run on interactive_queue = multiprocessing.Queue() batch_queue = multiprocessing.Queue() populate_queue(interactive_queue, INTERACTIVE_QUEUE) populate_queue(batch_queue, BATCH_QUEUE) # the workers will wait on event if the queues are exhausted new_job_event = multiprocessing.Event() # this lock will be used to edit the queue or new job event overall_lock = multiprocessing.Lock() # Setup watchdog observer that will watch the folders and run the handler on any # FileSystemEvents. This runs in a thread automatically. interactive_handler = NewFileHandler(interactive_queue,new_job_event,overall_lock) batch_handler = NewFileHandler(batch_queue,new_job_event,overall_lock) observer = Observer() observer.schedule(event_handler=interactive_handler, path=INTERACTIVE_QUEUE, recursive=False) observer.schedule(event_handler=batch_handler, path=BATCH_QUEUE, recursive=False) observer.start() # launch the worker threads processes = list() for i in range(0,num_workers): u = "untrusted" + str(i).zfill(2) p = multiprocessing.Process(target=worker_process,args=(interactive_queue,batch_queue,new_job_event,overall_lock,u)) p.start() processes.append(p) # main monitoring loop try: while True: alive = 0 for i in range(0,num_workers): if processes[i].is_alive: alive = alive+1 else: grade_items_logging.log_message(message="ERROR: process "+str(i)+" is not alive") if alive != num_workers: grade_items_logging.log_message(message="ERROR: #workers="+str(num_workers)+" != #alive="+str(alive)) #print ("workers= ",num_workers," alive=",alive) time.sleep(1) except KeyboardInterrupt: grade_items_logging.log_message(message="grade_scheduler.py keyboard interrupt") # just kill everything in this group id right now # NOTE: this may be a bug if the grandchildren have a different group id and not be killed os.kill(-os.getpid(), signal.SIGKILL) # run this to check if everything is dead # ps xao pid,ppid,pgid,sid,comm,user | grep untrust # everything's dead, including the main process so the rest of this will be ignored # but this was mostly working... # terminate the jobs for i in range(0,num_workers): processes[i].terminate() # wake up sleeping jobs new_job_event.set() # wait for them to join for i in range(0,num_workers): processes[i].join() # cleanup observer observer.stop() observer.join() grade_items_logging.log_message(message="grade_scheduler.py terminated")
def update_foreign_autograding_worker_json(name, entry): fd, tmp_json_path = tempfile.mkstemp() foreign_json = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO","autograding_worker.json") autograding_worker_to_ship = entry try: user = autograding_worker_to_ship[name]['username'] host = autograding_worker_to_ship[name]['address'] except Exception as e: print("ERROR: autograding_workers.json entry for {0} is malformatted. {1}".format(e, name)) grade_items_logging.log_message(JOB_ID, message="ERROR: autograding_workers.json entry for {0} is malformatted. {1}".format(e, name)) return #create a new temporary json with only the entry for the current machine. with open(tmp_json_path, 'w') as outfile: json.dump(autograding_worker_to_ship, outfile, sort_keys=True, indent=4) #if we are updating the current machine, we can just move the new json to the appropriate spot (no ssh needed) if host == "localhost": try: shutil.move(tmp_json_path,foreign_json) print("Successfully updated local autograding_TODO/autograding_worker.json") grade_items_logging.log_message(JOB_ID, message="Successfully updated local autograding_TODO/autograding_worker.json") except Exception as e: grade_items_logging.log_message(JOB_ID, message="ERROR: could not mv to local autograding_TODO/autograding_worker.json due to the following error: "+str(e)) print("ERROR: could not mv to local autograding_worker.json due to the following error: {0}".format(e)) finally: os.close(fd) #if we are updating a foreign machine, we must connect via ssh and use sftp to update it. else: #try to establish an ssh connection to the host try: ssh = paramiko.SSHClient() ssh.get_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname = host, username = user) except Exception as e: grade_items_logging.log_message(JOB_ID, message="ERROR: could not ssh to "+host+" due to following error: "+str(e)) print("ERROR: could not ssh to "+host+" due to following error: "+str(e)) return #try to copy the files over to the host try: sftp = ssh.open_sftp() sftp.put(tmp_json_path,foreign_json) sftp.close() print("Successfully forwarded autograding_worker.json to {0}".format(name)) grade_items_logging.log_message(JOB_ID, message="Successfully forwarded autograding_worker.json to {0}".format(name)) except Exception as e: grade_items_logging.log_message(JOB_ID, message="ERROR: could not sftp to foreign autograding_TODO/autograding_worker.json due to the following error: "+str(e)) print("ERROR: could sftp to foreign autograding_TODO/autograding_worker.json due to the following error: {0}".format(e)) success = False finally: os.close(fd) os.remove(tmp_json_path) sftp.close() ssh.close()
def launch_shippers(): # verify the hwcron user is running this script if not int(os.getuid()) == int(HWCRON_UID): raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user") grade_items_logging.log_message(JOB_ID, message="grade_scheduler.py launched") # Clean up old files from previous shipping/autograding (any # partially completed work will be re-done) for file_path in glob.glob(os.path.join(INTERACTIVE_QUEUE, "GRADING_*")): grade_items_logging.log_message(JOB_ID, message="Remove old queue file: " + file_path) os.remove(file_path) for file_path in glob.glob(os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO","unstrusted*")): grade_items_logging.log_message(JOB_ID, message="Remove autograding TODO file: " + file_path) os.remove(file_path) for file_path in glob.glob(os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE","*")): grade_items_logging.log_message(JOB_ID, message="Remove autograding DONE file: " + file_path) os.remove(file_path) # this lock will be used to edit the queue or new job event overall_lock = multiprocessing.Lock() # The names of the worker machines, the capabilities of each # worker machine, and the number of workers per machine are stored # in the autograding_workers json. try: autograding_workers_path = os.path.join(SUBMITTY_INSTALL_DIR, 'config', "autograding_workers.json") with open(autograding_workers_path, 'r') as infile: autograding_workers = json.load(infile) except Exception as e: raise SystemExit("ERROR: could not locate the autograding workers json: {0}".format(e)) # There must always be a primary machine, it may or may not have # autograding workers. if not "primary" in autograding_workers: raise SystemExit("ERROR: autograding_workers.json contained no primary machine.") # One (or more) of the machines must accept "default" jobs. default_present = False for name, machine in autograding_workers.items(): if "default" in machine["capabilities"]: default_present = True break if not default_present: raise SystemExit("ERROR: autograding_workers.json contained no machine with default capabilities") # Launch a shipper process for every work on the primary machine and each worker machine total_num_workers = 0 processes = list() for name, machine in autograding_workers.items(): try: which_machine=machine["address"] if which_machine != "localhost": if machine["username"] == "": raise SystemExit("ERROR: empty username for worker machine '" + which_machine + "'") which_machine = "{0}@{1}".format(machine["username"], machine["address"]) elif not machine["username"] == "": Raise('ERROR: username for primary (localhost) must be ""') num_workers_on_machine = machine["num_autograding_workers"] if num_workers_on_machine < 0: raise SystemExit("ERROR: num_workers_on_machine for '" + which_machine + "' must be non-negative.") my_capabilities = machine["capabilities"] except Exception as e: print("ERROR: autograding_workers.json entry for {0} contains an error: {1}".format(name, e)) grade_items_logging.log_message(JOB_ID, message="ERROR: autograding_workers.json entry for {0} contains an error: {1}".format(name,e)) continue # launch the shipper threads for i in range(0,num_workers_on_machine): u = "untrusted" + str(i).zfill(2) p = multiprocessing.Process(target=shipper_process,args=(name,which_machine,my_capabilities,u,overall_lock)) p.start() processes.append(p) total_num_workers += num_workers_on_machine # main monitoring loop try: while True: alive = 0 for i in range(0,total_num_workers): if processes[i].is_alive: alive = alive+1 else: grade_items_logging.log_message(JOB_ID, message="ERROR: process "+str(i)+" is not alive") if alive != total_num_workers: grade_items_logging.log_message(JOB_ID, message="ERROR: #shippers="+str(total_num_workers)+" != #alive="+str(alive)) #print ("shippers= ",total_num_workers," alive=",alive) time.sleep(1) except KeyboardInterrupt: grade_items_logging.log_message(JOB_ID, message="grade_scheduler.py keyboard interrupt") # just kill everything in this group id right now # NOTE: this may be a bug if the grandchildren have a different group id and not be killed os.kill(-os.getpid(), signal.SIGKILL) # run this to check if everything is dead # ps xao pid,ppid,pgid,sid,comm,user | grep untrust # everything's dead, including the main process so the rest of this will be ignored # but this was mostly working... # terminate the jobs for i in range(0,total_num_workers): processes[i].terminate() # wait for them to join for i in range(0,total_num_workers): processes[i].join() grade_items_logging.log_message(JOB_ID, message="grade_scheduler.py terminated")
def launch_workers(my_name, my_stats): num_workers = my_stats['num_autograding_workers'] # verify the hwcron user is running this script if not int(os.getuid()) == int(HWCRON_UID): raise SystemExit( "ERROR: the grade_item.py script must be run by the hwcron user") grade_items_logging.log_message(JOB_ID, message="grade_scheduler.py launched") # prepare a list of untrusted users to be used by the workers untrusted_users = multiprocessing.Queue() for i in range(num_workers): untrusted_users.put("untrusted" + str(i).zfill(2)) # launch the worker threads address = my_stats['address'] if address != 'localhost': which_machine = "{0}@{1}".format(my_stats['username'], address) else: which_machine = address my_server = my_stats['server_name'] processes = list() for i in range(0, num_workers): u = "untrusted" + str(i).zfill(2) p = multiprocessing.Process(target=worker_process, args=(which_machine, address, u, my_server)) p.start() processes.append(p) # main monitoring loop try: while True: alive = 0 for i in range(0, num_workers): if processes[i].is_alive: alive = alive + 1 else: grade_items_logging.log_message(JOB_ID, message="ERROR: process " + str(i) + " is not alive") if alive != num_workers: grade_items_logging.log_message(JOB_ID, message="ERROR: #workers=" + str(num_workers) + " != #alive=" + str(alive)) #print ("workers= ",num_workers," alive=",alive) time.sleep(1) except KeyboardInterrupt: grade_items_logging.log_message( JOB_ID, message="grade_scheduler.py keyboard interrupt") # just kill everything in this group id right now # NOTE: this may be a bug if the grandchildren have a different group id and not be killed os.kill(-os.getpid(), signal.SIGKILL) # run this to check if everything is dead # ps xao pid,ppid,pgid,sid,comm,user | grep untrust # everything's dead, including the main process so the rest of this will be ignored # but this was mostly working... # terminate the jobs for i in range(0, num_workers): processes[i].terminate() # wait for them to join for i in range(0, num_workers): processes[i].join() grade_items_logging.log_message(JOB_ID, message="grade_scheduler.py terminated")
def prepare_autograding_and_submission_zip(next_directory, next_to_grade): os.chdir(SUBMITTY_DATA_DIR) # -------------------------------------------------------- # figure out what we're supposed to grade & error checking obj = load_queue_file_obj(next_directory, next_to_grade) partial_path = os.path.join(obj["gradeable"], obj["who"], str(obj["version"])) item_name = os.path.join(obj["semester"], obj["course"], "submissions", partial_path) submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name) if not os.path.isdir(submission_path): grade_items_logging.log_message( message="ERROR: the submission directory does not exist" + submission_path) raise RuntimeError("ERROR: the submission directory does not exist", submission_path) print("pid", os.getpid(), "GRADE THIS", submission_path) is_vcs, vcs_type, vcs_base_url, vcs_subdirectory = get_vcs_info( SUBMITTY_DATA_DIR, obj["semester"], obj["course"], obj["gradeable"], obj["who"], obj["team"]) is_batch_job = next_directory == BATCH_QUEUE is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE" queue_time = get_queue_time(next_directory, next_to_grade) queue_time_longstring = dateutils.write_submitty_date(queue_time) grading_began = dateutils.get_current_time() waittime = (grading_began - queue_time).total_seconds() grade_items_logging.log_message(is_batch_job, "zip", item_name, "wait:", waittime, "") # -------------------------------------------------------------------- # MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE tmp = tempfile.mkdtemp() tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING") os.mkdir(tmp_autograding) tmp_submission = os.path.join(tmp, "TMP_SUBMISSION") os.mkdir(tmp_submission) # -------------------------------------------------------- # various paths provided_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "provided_code", obj["gradeable"]) test_input_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_input", obj["gradeable"]) test_output_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "test_output", obj["gradeable"]) custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "custom_validation_code", obj["gradeable"]) bin_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "bin", obj["gradeable"]) form_json_config = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "form", "form_" + obj["gradeable"] + ".json") complete_config = os.path.join( SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config", "complete_config", "complete_config_" + obj["gradeable"] + ".json") copytree_if_exists(provided_code_path, os.path.join(tmp_autograding, "provided_code")) copytree_if_exists(test_input_path, os.path.join(tmp_autograding, "test_input")) copytree_if_exists(test_output_path, os.path.join(tmp_autograding, "test_output")) copytree_if_exists(custom_validation_code_path, os.path.join(tmp_autograding, "custom_validation_code")) copytree_if_exists(bin_path, os.path.join(tmp_autograding, "bin")) shutil.copy(form_json_config, os.path.join(tmp_autograding, "form.json")) shutil.copy(complete_config, os.path.join(tmp_autograding, "complete_config.json")) checkout_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "checkout", partial_path) results_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "results", partial_path) # grab a copy of the current history.json file (if it exists) history_file = os.path.join(results_path, "history.json") history_file_tmp = "" if os.path.isfile(history_file): filehandle, history_file_tmp = tempfile.mkstemp() shutil.copy(history_file, history_file_tmp) shutil.copy(history_file, os.path.join(tmp_submission, "history.json")) # get info from the gradeable config file with open(complete_config, 'r') as infile: complete_config_obj = json.load(infile) checkout_subdirectory = complete_config_obj["autograding"].get( "use_checkout_subdirectory", "") checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory) queue_file = os.path.join(next_directory, next_to_grade) # switch to tmp directory os.chdir(tmp) # make the logs directory tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs") os.makedirs(tmp_logs) # 'touch' a file in the logs folder open(os.path.join(tmp_logs, "overall.txt"), 'a') # grab the submission time with open(os.path.join(submission_path, ".submit.timestamp")) as submission_time_file: submission_string = submission_time_file.read().rstrip() submission_datetime = dateutils.read_submitty_date(submission_string) # -------------------------------------------------------------------- # CHECKOUT THE STUDENT's REPO if is_vcs: # is vcs_subdirectory standalone or should it be combined with base_url? if vcs_subdirectory[0] == '/' or '://' in vcs_subdirectory: vcs_path = vcs_subdirectory else: if '://' in vcs_base_url: vcs_path = urllib.parse.urljoin(vcs_base_url, vcs_subdirectory) else: vcs_path = os.path.join(vcs_base_url, vcs_subdirectory) with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f: print("====================================\nVCS CHECKOUT", file=f) print('vcs_base_url', vcs_base_url, file=f) print('vcs_subdirectory', vcs_subdirectory, file=f) print('vcs_path', vcs_path, file=f) print(['/usr/bin/git', 'clone', vcs_path, checkout_path], file=f) # cleanup the previous checkout (if it exists) shutil.rmtree(checkout_path, ignore_errors=True) os.makedirs(checkout_path, exist_ok=True) subprocess.call(['/usr/bin/git', 'clone', vcs_path, checkout_path]) os.chdir(checkout_path) # determine which version we need to checkout what_version = subprocess.check_output([ 'git', 'rev-list', '-n', '1', '--before="' + submission_string + '"', 'master' ]) what_version = str(what_version.decode('utf-8')).rstrip() if what_version == "": # oops, pressed the grade button before a valid commit shutil.rmtree(checkout_path, ignore_errors=True) else: # and check out the right version subprocess.call(['git', 'checkout', '-b', 'grade', what_version]) os.chdir(tmp) subprocess.call(['ls', '-lR', checkout_path], stdout=open(tmp_logs + "/overall.txt", 'a')) copytree_if_exists(submission_path, os.path.join(tmp_submission, "submission")) copytree_if_exists(checkout_path, os.path.join(tmp_submission, "checkout")) obj["queue_time"] = queue_time_longstring obj["is_batch_job"] = is_batch_job obj["waittime"] = waittime with open(os.path.join(tmp_submission, "queue_file.json"), 'w') as outfile: json.dump(obj, outfile, sort_keys=True, indent=4, separators=(',', ': ')) grading_began_longstring = dateutils.write_submitty_date(grading_began) with open(os.path.join(tmp_submission, ".grading_began"), 'w') as f: print(grading_began_longstring, file=f) # zip up autograding & submission folders my_autograding_zip_file = tempfile.mkstemp()[1] my_submission_zip_file = tempfile.mkstemp()[1] zip_my_directory(tmp_autograding, my_autograding_zip_file) zip_my_directory(tmp_submission, my_submission_zip_file) # cleanup shutil.rmtree(tmp_autograding) shutil.rmtree(tmp_submission) shutil.rmtree(tmp) return (my_autograding_zip_file, my_submission_zip_file)
def unpack_job(which_machine,which_untrusted,next_directory,next_to_grade): # verify the hwcron user is running this script if not int(os.getuid()) == int(HWCRON_UID): grade_items_logging.log_message(JOB_ID, message="ERROR: must be run by hwcron") raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user") if which_machine == 'localhost': address = which_machine else: address = which_machine.split('@')[1] fully_qualified_domain_name = socket.getfqdn() servername_workername = "{0}_{1}".format(fully_qualified_domain_name, address) target_results_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_results.zip") target_done_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_queue.json") if which_machine == "localhost": if not os.path.exists(target_done_queue_file): return False else: local_done_queue_file = target_done_queue_file local_results_zip = target_results_zip else: user, host = which_machine.split("@") ssh = paramiko.SSHClient() ssh.get_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: ssh.connect(hostname = host, username = user) sftp = ssh.open_sftp() fd1, local_done_queue_file = tempfile.mkstemp() fd2, local_results_zip = tempfile.mkstemp() #remote path first, then local. sftp.get(target_done_queue_file, local_done_queue_file) sftp.get(target_results_zip, local_results_zip) #Because get works like cp rather tnan mv, we have to clean up. sftp.remove(target_done_queue_file) sftp.remove(target_results_zip) success = True #This is the normal case (still grading on the other end) so we don't need to print anything. except FileNotFoundError: os.remove(local_results_zip) os.remove(local_done_queue_file) success = False #In this more general case, we do want to print what the error was. #TODO catch other types of exception as we identify them. except Exception as e: grade_items_logging.log_message(JOB_ID, message="ERROR: Could not retrieve the file from the foreign machine "+str(e)) print("ERROR: Could not retrieve the file from the foreign machine.\nERROR: {0}".format(e)) os.remove(local_results_zip) os.remove(local_done_queue_file) success = False finally: os.close(fd1) os.close(fd2) sftp.close() ssh.close() if not success: return False # archive the results of grading try: grade_item.unpack_grading_results_zip(which_machine,which_untrusted,local_results_zip) except: grade_items_logging.log_message(JOB_ID,jobname=next_to_grade,message="ERROR: Exception when unpacking zip") with contextlib.suppress(FileNotFoundError): os.remove(local_results_zip) with contextlib.suppress(FileNotFoundError): os.remove(local_done_queue_file) return True
def prepare_job(my_name,which_machine,which_untrusted,next_directory,next_to_grade): # verify the hwcron user is running this script if not int(os.getuid()) == int(HWCRON_UID): grade_items_logging.log_message(JOB_ID, message="ERROR: must be run by hwcron") raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user") if which_machine == 'localhost': address = which_machine else: address = which_machine.split('@')[1] # prepare the zip files try: autograding_zip_tmp,submission_zip_tmp = grade_item.prepare_autograding_and_submission_zip(which_machine,which_untrusted,next_directory,next_to_grade) fully_qualified_domain_name = socket.getfqdn() servername_workername = "{0}_{1}".format(fully_qualified_domain_name, address) autograding_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_autograding.zip") submission_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_submission.zip") todo_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_queue.json") with open(next_to_grade, 'r') as infile: queue_obj = json.load(infile) queue_obj["which_untrusted"] = which_untrusted queue_obj["which_machine"] = which_machine queue_obj["ship_time"] = dateutils.write_submitty_date(microseconds=True) except Exception as e: grade_items_logging.log_message(JOB_ID, message="ERROR: failed preparing submission zip or accessing next to grade "+str(e)) print("ERROR: failed preparing submission zip or accessing next to grade ", e) return False if address == "localhost": try: shutil.move(autograding_zip_tmp,autograding_zip) shutil.move(submission_zip_tmp,submission_zip) with open(todo_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) except Exception as e: grade_items_logging.log_message(JOB_ID, message="ERROR: could not move files due to the following error: "+str(e)) print("ERROR: could not move files due to the following error: {0}".format(e)) return False else: try: user, host = which_machine.split("@") ssh = paramiko.SSHClient() ssh.get_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname = host, username = user) sftp = ssh.open_sftp() sftp.put(autograding_zip_tmp,autograding_zip) sftp.put(submission_zip_tmp,submission_zip) with open(todo_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) sftp.put(todo_queue_file, todo_queue_file) os.remove(todo_queue_file) print("Successfully forwarded files to {0}".format(my_name)) success = True except Exception as e: grade_items_logging.log_message(JOB_ID, message="ERROR: could not move files due to the following error: "+str(e)) print("Could not move files due to the following error: {0}".format(e)) success = False finally: sftp.close() ssh.close() os.remove(autograding_zip_tmp) os.remove(submission_zip_tmp) return success return True
def worker_process(which_machine, address, which_untrusted, my_server): # verify the hwcron user is running this script if not int(os.getuid()) == int(HWCRON_UID): grade_items_logging.log_message(JOB_ID, message="ERROR: must be run by hwcron") raise SystemExit( "ERROR: the grade_item.py script must be run by the hwcron user") # ignore keyboard interrupts in the worker processes signal.signal(signal.SIGINT, signal.SIG_IGN) counter = 0 servername_workername = "{0}_{1}".format(my_server, address) autograding_zip = os.path.join( SUBMITTY_DATA_DIR, "autograding_TODO", servername_workername + "_" + which_untrusted + "_autograding.zip") submission_zip = os.path.join( SUBMITTY_DATA_DIR, "autograding_TODO", servername_workername + "_" + which_untrusted + "_submission.zip") todo_queue_file = os.path.join( SUBMITTY_DATA_DIR, "autograding_TODO", servername_workername + "_" + which_untrusted + "_queue.json") while True: if os.path.exists(todo_queue_file): try: results_zip_tmp = grade_item.grade_from_zip( autograding_zip, submission_zip, which_untrusted) results_zip = os.path.join( SUBMITTY_DATA_DIR, "autograding_DONE", servername_workername + "_" + which_untrusted + "_results.zip") done_queue_file = os.path.join( SUBMITTY_DATA_DIR, "autograding_DONE", servername_workername + "_" + which_untrusted + "_queue.json") #move doesn't inherit the permissions of the destination directory. Copyfile does. shutil.copyfile(results_zip_tmp, results_zip) os.remove(results_zip_tmp) with open(todo_queue_file, 'r') as infile: queue_obj = json.load(infile) queue_obj["done_time"] = dateutils.write_submitty_date( microseconds=True) with open(done_queue_file, 'w') as outfile: json.dump(queue_obj, outfile, sort_keys=True, indent=4) except Exception as e: grade_items_logging.log_message( JOB_ID, message="ERROR attempting to unzip graded item: " + which_machine + " " + which_untrusted + " exception " + repr(e)) with contextlib.suppress(FileNotFoundError): os.remove(autograding_zip) with contextlib.suppress(FileNotFoundError): os.remove(submission_zip) with contextlib.suppress(FileNotFoundError): os.remove(todo_queue_file) counter = 0 else: if counter >= 10: print(which_machine, which_untrusted, "wait") counter = 0 counter += 1 time.sleep(1)