def unpack_job(which_machine,which_untrusted,next_directory,next_to_grade): # variables needed for logging obj = packer_unpacker.load_queue_file_obj(JOB_ID,next_directory,next_to_grade) partial_path = os.path.join(obj["gradeable"],obj["who"],str(obj["version"])) item_name = os.path.join(obj["semester"],obj["course"],"submissions",partial_path) is_batch = "regrade" in obj and obj["regrade"] # verify the DAEMON_USER is running this script if not int(os.getuid()) == int(DAEMON_UID): grade_items_logging.log_message(JOB_ID, message="ERROR: must be run by DAEMON_USER") raise SystemExit("ERROR: the grade_item.py script must be run by the DAEMON_USER") if which_machine == 'localhost': address = which_machine else: address = which_machine.split('@')[1] fully_qualified_domain_name = socket.getfqdn() servername_workername = "{0}_{1}".format(fully_qualified_domain_name, address) target_results_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_results.zip") target_done_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_queue.json") if which_machine == "localhost": if not os.path.exists(target_done_queue_file): return False else: local_done_queue_file = target_done_queue_file local_results_zip = target_results_zip else: user, host = which_machine.split("@") ssh = paramiko.SSHClient() ssh.get_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: ssh.connect(hostname = host, username = user) sftp = ssh.open_sftp() fd1, local_done_queue_file = tempfile.mkstemp() fd2, local_results_zip = tempfile.mkstemp() #remote path first, then local. sftp.get(target_done_queue_file, local_done_queue_file) sftp.get(target_results_zip, local_results_zip) #Because get works like cp rather tnan mv, we have to clean up. sftp.remove(target_done_queue_file) sftp.remove(target_results_zip) success = True #This is the normal case (still grading on the other end) so we don't need to print anything. except FileNotFoundError: os.remove(local_results_zip) os.remove(local_done_queue_file) success = False #In this more general case, we do want to print what the error was. #TODO catch other types of exception as we identify them. except Exception as e: grade_items_logging.log_message(JOB_ID, message="ERROR: Could not retrieve the file from the foreign machine "+str(e)) print("ERROR: Could not retrieve the file from the foreign machine.\nERROR: {0}".format(e)) os.remove(local_results_zip) os.remove(local_done_queue_file) success = False finally: os.close(fd1) os.close(fd2) sftp.close() ssh.close() if not success: return False # archive the results of grading try: success = packer_unpacker.unpack_grading_results_zip(which_machine,which_untrusted,local_results_zip) except: grade_items_logging.log_message(JOB_ID,jobname=item_name,message="ERROR: Exception when unpacking zip") with contextlib.suppress(FileNotFoundError): os.remove(local_results_zip) success = False with contextlib.suppress(FileNotFoundError): os.remove(local_done_queue_file) msg = "Unpacked job from " + which_machine if success else "ERROR: failure returned from worker machine" print(msg) grade_items_logging.log_message(JOB_ID, jobname=item_name, which_untrusted=which_untrusted, is_batch=is_batch, message=msg) return True
def unpack_job(which_machine,which_untrusted,next_directory,next_to_grade): # variables needed for logging obj = packer_unpacker.load_queue_file_obj(JOB_ID,next_directory,next_to_grade) if "generate_output" not in obj: partial_path = os.path.join(obj["gradeable"],obj["who"],str(obj["version"])) item_name = os.path.join(obj["semester"],obj["course"],"submissions",partial_path) elif obj["generate_output"]: item_name = os.path.join(obj["semester"],obj["course"],"generated_output") is_batch = "regrade" in obj and obj["regrade"] # verify the DAEMON_USER is running this script if not int(os.getuid()) == int(DAEMON_UID): autograding_utils.log_message(AUTOGRADING_LOG_PATH, JOB_ID, message="ERROR: must be run by DAEMON_USER") raise SystemExit("ERROR: the submitty_autograding_shipper.py script must be run by the DAEMON_USER") if which_machine == 'localhost': address = which_machine else: address = which_machine.split('@')[1] fully_qualified_domain_name = socket.getfqdn() servername_workername = "{0}_{1}".format(fully_qualified_domain_name, address) target_results_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_results.zip") target_done_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_queue.json") if which_machine == "localhost": if not os.path.exists(target_done_queue_file): return False else: local_done_queue_file = target_done_queue_file local_results_zip = target_results_zip else: ssh = sftp = fd1 = fd2 = local_done_queue_file = local_results_zip = None try: user, host = which_machine.split("@") ssh = establish_ssh_connection(which_machine, user, host) sftp = ssh.open_sftp() fd1, local_done_queue_file = tempfile.mkstemp() fd2, local_results_zip = tempfile.mkstemp() #remote path first, then local. sftp.get(target_done_queue_file, local_done_queue_file) sftp.get(target_results_zip, local_results_zip) #Because get works like cp rather tnan mv, we have to clean up. sftp.remove(target_done_queue_file) sftp.remove(target_results_zip) success = True #This is the normal case (still grading on the other end) so we don't need to print anything. except (socket.timeout, TimeoutError) as e: success = False except FileNotFoundError: # Remove results files for var in [local_results_zip, local_done_queue_file]: if var: with contextlib.suppress(FileNotFoundError): os.remove(var) success = False #In this more general case, we do want to print what the error was. #TODO catch other types of exception as we identify them. except Exception as e: autograding_utils.log_stack_trace(AUTOGRADING_STACKTRACE_PATH, job_id=JOB_ID, trace=traceback.format_exc()) autograding_utils.log_message(AUTOGRADING_LOG_PATH, JOB_ID, message="ERROR: Could not retrieve the file from the foreign machine "+str(e)) print("ERROR: Could not retrieve the file from the foreign machine.\nERROR: {0}".format(e)) # Remove results files for var in [local_results_zip, local_done_queue_file]: if var: with contextlib.suppress(FileNotFoundError): os.remove(var) success = False finally: # Close SSH connections for var in [sftp, ssh]: if var: var.close() # Close file descriptors for var in [fd1, fd2]: if var: try: os.close(var) except Exception: pass if not success: return False # archive the results of grading try: success = packer_unpacker.unpack_grading_results_zip(which_machine,which_untrusted,local_results_zip) except: autograding_utils.log_stack_trace(AUTOGRADING_STACKTRACE_PATH, job_id=JOB_ID, trace=traceback.format_exc()) autograding_utils.log_message(AUTOGRADING_LOG_PATH, JOB_ID,jobname=item_name,message="ERROR: Exception when unpacking zip. For more details, see traces entry.") with contextlib.suppress(FileNotFoundError): os.remove(local_results_zip) success = False with contextlib.suppress(FileNotFoundError): os.remove(local_done_queue_file) msg = "Unpacked job from " + which_machine if success else "ERROR: failure returned from worker machine" print(msg) autograding_utils.log_message(AUTOGRADING_LOG_PATH, JOB_ID, jobname=item_name, which_untrusted=which_untrusted, is_batch=is_batch, message=msg) return True
def unpack_job(which_machine,which_untrusted,next_directory,next_to_grade): # variables needed for logging obj = packer_unpacker.load_queue_file_obj(JOB_ID,next_directory,next_to_grade) partial_path = os.path.join(obj["gradeable"],obj["who"],str(obj["version"])) item_name = os.path.join(obj["semester"],obj["course"],"submissions",partial_path) is_batch = "regrade" in obj and obj["regrade"] # verify the DAEMON_USER is running this script if not int(os.getuid()) == int(DAEMON_UID): grade_items_logging.log_message(JOB_ID, message="ERROR: must be run by DAEMON_USER") raise SystemExit("ERROR: the grade_item.py script must be run by the DAEMON_USER") if which_machine == 'localhost': address = which_machine else: address = which_machine.split('@')[1] fully_qualified_domain_name = socket.getfqdn() servername_workername = "{0}_{1}".format(fully_qualified_domain_name, address) target_results_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_results.zip") target_done_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_queue.json") if which_machine == "localhost": if not os.path.exists(target_done_queue_file): return False else: local_done_queue_file = target_done_queue_file local_results_zip = target_results_zip else: user, host = which_machine.split("@") ssh = paramiko.SSHClient() ssh.get_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: ssh.connect(hostname = host, username = user, timeout=5) sftp = ssh.open_sftp() fd1, local_done_queue_file = tempfile.mkstemp() fd2, local_results_zip = tempfile.mkstemp() #remote path first, then local. sftp.get(target_done_queue_file, local_done_queue_file) sftp.get(target_results_zip, local_results_zip) #Because get works like cp rather tnan mv, we have to clean up. sftp.remove(target_done_queue_file) sftp.remove(target_results_zip) success = True #This is the normal case (still grading on the other end) so we don't need to print anything. except FileNotFoundError: os.remove(local_results_zip) os.remove(local_done_queue_file) success = False #In this more general case, we do want to print what the error was. #TODO catch other types of exception as we identify them. except Exception as e: grade_items_logging.log_stack_trace(job_id=JOB_ID, trace=traceback.format_exc()) grade_items_logging.log_message(JOB_ID, message="ERROR: Could not retrieve the file from the foreign machine "+str(e)) print("ERROR: Could not retrieve the file from the foreign machine.\nERROR: {0}".format(e)) os.remove(local_results_zip) os.remove(local_done_queue_file) success = False finally: os.close(fd1) os.close(fd2) sftp.close() ssh.close() if not success: return False # archive the results of grading try: success = packer_unpacker.unpack_grading_results_zip(which_machine,which_untrusted,local_results_zip) except: grade_items_logging.log_stack_trace(job_id=JOB_ID, trace=traceback.format_exc()) grade_items_logging.log_message(JOB_ID,jobname=item_name,message="ERROR: Exception when unpacking zip. For more details, see traces entry.") with contextlib.suppress(FileNotFoundError): os.remove(local_results_zip) success = False with contextlib.suppress(FileNotFoundError): os.remove(local_done_queue_file) msg = "Unpacked job from " + which_machine if success else "ERROR: failure returned from worker machine" print(msg) grade_items_logging.log_message(JOB_ID, jobname=item_name, which_untrusted=which_untrusted, is_batch=is_batch, message=msg) return True