def shipper_process(my_name, which_machine,my_capabilities,which_untrusted,overall_lock):
"""
Each shipper process spins in a loop, looking for a job that
matches the capabilities of this machine, and then oversees the
autograding of that job. Interactive jobs are prioritized over
batch (regrade) jobs. If no jobs are available, the shipper waits
on an event editing one of the queues.
"""
# ignore keyboard interrupts in the shipper processes
signal.signal(signal.SIGINT, signal.SIG_IGN)
counter=0
while True:
try:
my_job = get_job(my_name,which_machine,my_capabilities,which_untrusted,overall_lock)
if not my_job == "":
counter=0
grade_queue_file(my_name,which_machine,which_untrusted,os.path.join(INTERACTIVE_QUEUE,my_job))
continue
else:
if counter == 0 or counter >= 10:
print (my_name,which_untrusted,"no available job")
counter=0
counter+=1
time.sleep(1)
except Exception as e:
my_message = "ERROR in get_job " + which_machine + " " + which_untrusted + " " + str(e)
print (my_message)
grade_items_logging.log_message(JOB_ID, message=my_message)
time.sleep(1)
def copy_contents_into(source,target,tmp_logs):
if not os.path.isdir(target):
grade_items_logging.log_message("ERROR: the target directory does not exist " + target)
raise SystemExit("ERROR: the target directory does not exist '", target, "'")
if os.path.isdir(source):
for item in os.listdir(source):
if os.path.isdir(os.path.join(source,item)):
if os.path.isdir(os.path.join(target,item)):
# recurse
copy_contents_into(os.path.join(source,item),os.path.join(target,item),tmp_logs)
elif os.path.isfile(os.path.join(target,item)):
grade_items_logging.log_message("ERROR: the target subpath is a file not a directory '" + os.path.join(target,item) + "'")
raise SystemExit("ERROR: the target subpath is a file not a directory '", os.path.join(target,item), "'")
else:
# copy entire subtree
shutil.copytree(os.path.join(source,item),os.path.join(target,item))
else:
if os.path.exists(os.path.join(target,item)):
with open(os.path.join(tmp_logs,"overall.txt"),'a') as f:
print ("\nWARNING: REMOVING DESTINATION FILE" , os.path.join(target,item),
" THEN OVERWRITING: ", os.path.join(source,item), "\n", file=f)
os.remove(os.path.join(target,item))
try:
shutil.copy(os.path.join(source,item),target)
except:
raise SystemExit("ERROR COPYING FILE: " + os.path.join(source,item) + " -> " + os.path.join(target,item))
def get_submission_path(next_directory,next_to_grade):
queue_file = os.path.join(next_directory,next_to_grade)
if not os.path.isfile(queue_file):
grade_items_logging.log_message("ERROR: the file does not exist " + queue_file)
raise SystemExit("ERROR: the file does not exist",queue_file)
with open(queue_file, 'r') as infile:
obj = json.load(infile)
return obj
def load_queue_file_obj(next_directory, next_to_grade):
queue_file = os.path.join(next_directory, next_to_grade)
if not os.path.isfile(queue_file):
grade_items_logging.log_message(
message="ERROR: the file does not exist " + queue_file)
raise RuntimeError("ERROR: the file does not exist", queue_file)
with open(queue_file, 'r') as infile:
obj = json.load(infile)
return obj
def populate_queue(queue, folder):
"""
Populate a queue with all files in folder. We first scan the folder to check for any
"GRADING_*" and clean them up, and then add the remaining files to the queue, sorted
by creation time.
:param queue: multiprocessing.queues.Queue
:param folder: string representing the path to the folder to add files from
"""
for file_path in glob.glob(os.path.join(folder, "GRADING_*")):
grade_items_logging.log_message(message="Remove old queue file: " + file_path)
os.remove(file_path)
# Grab all the files currently in the folder, sorted by creation
# time, and put them in the queue to be graded
files = glob.glob(os.path.join(folder, "*"))
files.sort(key=os.path.getctime)
for f in files:
queue.put(os.path.join(folder, f))
def unpack_grading_results_zip(which_machine, which_untrusted,
my_results_zip_file):
os.chdir(SUBMITTY_DATA_DIR)
queue_obj = unzip_queue_file(my_results_zip_file)
job_id = queue_obj["job_id"]
partial_path = os.path.join(queue_obj["gradeable"], queue_obj["who"],
str(queue_obj["version"]))
item_name = os.path.join(queue_obj["semester"], queue_obj["course"],
"submissions", partial_path)
results_path = os.path.join(SUBMITTY_DATA_DIR, "courses",
queue_obj["semester"], queue_obj["course"],
"results", partial_path)
# clean out all of the old files if this is a re-run
shutil.rmtree(results_path, ignore_errors=True)
# create the directory (and the full path if it doesn't already exist)
os.makedirs(results_path)
# unzip the file & clean up
unzip_this_file(my_results_zip_file, results_path)
os.remove(my_results_zip_file)
# add information to the database
insert_database_version_data.insert_to_database(
queue_obj["semester"], queue_obj["course"], queue_obj["gradeable"],
queue_obj["user"], queue_obj["team"], queue_obj["who"],
True if queue_obj["is_team"] else False, str(queue_obj["version"]))
submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name)
is_batch_job = queue_obj["regrade"]
gradingtime = queue_obj["gradingtime"]
grade_result = queue_obj["grade_result"]
print(which_machine, which_untrusted, "unzip", item_name, " in ",
int(gradingtime), " seconds")
grade_items_logging.log_message(job_id, is_batch_job, "unzip", item_name,
"grade:", gradingtime, grade_result)
def grade_queue_file(queue_file,which_untrusted):
"""
Grades a single item in one of the queues.
:param queue_file: file path pointing to the file we want to operate one.
"""
my_dir,my_file=os.path.split(queue_file)
pid = os.getpid()
directory = os.path.dirname(os.path.realpath(queue_file))
name = os.path.basename(os.path.realpath(queue_file))
grading_file = os.path.join(directory, "GRADING_" + name)
open(os.path.join(grading_file), "w").close()
#untrusted = multiprocessing.current_process().untrusted
try:
grade_item.just_grade_item(my_dir, queue_file, which_untrusted)
except Exception as e:
print ("ERROR attempting to grade item: ", queue_file, " exception=",e)
grade_items_logging.log_message(message="ERROR attempting to grade item: " + queue_file + " exception " + repr(e))
# note: not necessary to acquire lock for these statements, but
# make sure you remove the queue file, then the grading file
try:
os.remove(queue_file)
except:
print ("ERROR attempting to remove queue file: ", queue_file)
grade_items_logging.log_message(message="ERROR attempting to remove queue file: " + queue_file)
try:
os.remove(grading_file)
except:
print ("ERROR attempting to remove grading file: ", grading_file)
grade_items_logging.log_message(message="ERROR attempting to remove grading file: " + grading_file)
def just_grade_item(next_directory, next_to_grade, which_untrusted):
# verify the hwcron user is running this script
if not int(os.getuid()) == int(HWCRON_UID):
grade_items_logging.log_message(message="ERROR: must be run by hwcron")
raise SystemExit(
"ERROR: the grade_item.py script must be run by the hwcron user")
# prepare the zip files
try:
autograding_zip, submission_zip = prepare_autograding_and_submission_zip(
next_directory, next_to_grade)
except:
grade_items_logging.log_message(
jobname=next_to_grade,
message=
"ERROR: Exception when preparing autograding and submission zip")
return
# actually do the grading (this step could be shipped to another machine)
try:
results_zip = grade_from_zip(autograding_zip, submission_zip,
which_untrusted)
except:
grade_items_logging.log_message(
jobname=next_to_grade,
message="ERROR: Exception when grading from zip")
with contextlib.suppress(FileNotFoundError):
os.remove(autograding_zip)
with contextlib.suppress(FileNotFoundError):
os.remove(submission_zip)
return
# archive the results of grading
try:
unpack_grading_results_zip(results_zip)
except:
grade_items_logging.log_message(
jobname=next_to_grade,
message="ERROR: Exception when unpacking zip")
with contextlib.suppress(FileNotFoundError):
os.remove(results_zip)
return
def grade_queue_file(my_name, which_machine,which_untrusted,queue_file):
"""
Oversees the autograding of single item from the queue
:param queue_file: details of what to grade
:param which_machine: name of machine to send this job to (might be "localhost")
:param which_untrusted: specific untrusted user for this autograding job
"""
my_dir,my_file=os.path.split(queue_file)
pid = os.getpid()
directory = os.path.dirname(os.path.realpath(queue_file))
name = os.path.basename(os.path.realpath(queue_file))
grading_file = os.path.join(directory, "GRADING_" + name)
#TODO: breach which_machine into id, address, and passphrase.
try:
# prepare the job
shipper_counter=0
while not prepare_job(my_name,which_machine, which_untrusted, my_dir, queue_file):
shipper_counter = 0
time.sleep(1)
if shipper_counter >= 10:
prints(my_name, which_untrusted, "shipper prep loop: ",queue_file)
shipper_counter=0
# then wait for grading to be completed
shipper_counter=0
while not unpack_job(which_machine, which_untrusted, my_dir, queue_file):
shipper_counter+=1
time.sleep(1)
if shipper_counter >= 10:
print (my_name,which_untrusted,"shipper wait for grade: ",queue_file)
shipper_counter=0
except Exception as e:
print (my_name, " ERROR attempting to grade item: ", queue_file, " exception=",str(e))
grade_items_logging.log_message(JOB_ID, message=str(my_name)+" ERROR attempting to grade item: " + queue_file + " exception " + repr(e))
# note: not necessary to acquire lock for these statements, but
# make sure you remove the queue file, then the grading file
try:
os.remove(queue_file)
except Exception as e:
print (my_name, " ERROR attempting to remove queue file: ", queue_file, " exception=",str(e))
grade_items_logging.log_message(JOB_ID, message=str(my_name)+" ERROR attempting to remove queue file: " + queue_file + " exception=" + str(e))
try:
os.remove(grading_file)
except Exception as e:
print (my_name, " ERROR attempting to remove grading file: ", grading_file, " exception=",str(e))
grade_items_logging.log_message(JOB_ID, message=str(my_name)+" ERROR attempting to remove grading file: " + grading_file + " exception=" + str(e))
def get_job(my_name,which_machine,my_capabilities,which_untrusted,overall_lock):
"""
Picks a job from the queue
:param overall_lock: a lock on the directory containing all queue files
"""
time_get_job_begin = dateutils.get_current_time()
overall_lock.acquire()
folder= INTERACTIVE_QUEUE
# Grab all the files currently in the folder, sorted by creation
# time, and put them in the queue to be graded
files = glob.glob(os.path.join(folder, "*"))
files_and_times = list()
for f in files:
try:
my_time = os.path.getctime(f)
except:
continue
tup = (f, my_time)
files_and_times.append(tup)
files_and_times = sorted(files_and_times, key=operator.itemgetter(1))
my_job=""
for full_path_file, file_time in files_and_times:
# get the file name (without the path)
just_file = full_path_file[len(folder)+1:]
# skip items that are already being graded
if (just_file[0:8]=="GRADING_"):
continue
grading_file = os.path.join(folder,"GRADING_"+just_file)
if grading_file in files:
continue
# found something to do
try:
with open(full_path_file, 'r') as infile:
queue_obj = json.load(infile)
except:
continue
#Check to make sure that we are capable of grading this submission
required_capabilities = queue_obj["required_capabilities"]
if not required_capabilities in my_capabilities:
continue
# prioritize interactive jobs over (batch) regrades
# if you've found an interactive job, exit early (since they are sorted by timestamp)
if not "regrade" in queue_obj or not queue_obj["regrade"]:
my_job = just_file
break
# otherwise it's a regrade, and if we don't already have a
# job, take it, but we have to search the rest of the list
if my_job == "":
my_job = just_file
if not my_job == "":
grading_file = os.path.join(folder, "GRADING_" + my_job)
# create the grading file
open(os.path.join(grading_file), "w").close()
overall_lock.release()
time_get_job_end = dateutils.get_current_time()
time_delta = time_get_job_end-time_get_job_begin
if time_delta > datetime.timedelta(milliseconds=100):
print (my_name, " WARNING: submitty_autograding shipper get_job time ", time_delta)
grade_items_logging.log_message(JOB_ID, message=str(my_name)+" WARNING: submitty_autograding shipper get_job time "+str(time_delta))
return my_job
def grade_from_zip(my_autograding_zip_file, my_submission_zip_file,
which_untrusted):
os.chdir(SUBMITTY_DATA_DIR)
tmp = os.path.join("/var/local/submitty/autograding_tmp/", which_untrusted,
"tmp")
# clean up old usage of this directory
shutil.rmtree(tmp, ignore_errors=True)
os.makedirs(tmp)
# unzip autograding and submission folders
tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING")
tmp_submission = os.path.join(tmp, "TMP_SUBMISSION")
unzip_this_file(my_autograding_zip_file, tmp_autograding)
unzip_this_file(my_submission_zip_file, tmp_submission)
os.remove(my_autograding_zip_file)
os.remove(my_submission_zip_file)
tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs")
queue_file = os.path.join(tmp_submission, "queue_file.json")
with open(queue_file, 'r') as infile:
queue_obj = json.load(infile)
queue_time_longstring = queue_obj["queue_time"]
waittime = queue_obj["waittime"]
is_batch_job = queue_obj["is_batch_job"]
is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE"
partial_path = os.path.join(queue_obj["gradeable"], queue_obj["who"],
str(queue_obj["version"]))
item_name = os.path.join(queue_obj["semester"], queue_obj["course"],
"submissions", partial_path)
grade_items_logging.log_message(is_batch_job, which_untrusted, item_name,
"wait:", waittime, "")
# --------------------------------------------------------------------
# START DOCKER
# WIP: This option file facilitated testing...
#USE_DOCKER = os.path.isfile("/tmp/use_docker")
#use_docker_string="grading begins, using DOCKER" if USE_DOCKER else "grading begins (not using docker)"
#grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,message=use_docker_string)
container = None
if USE_DOCKER:
container = subprocess.check_output([
'docker', 'run', '-t', '-d', '-v', tmp + ':' + tmp, 'ubuntu:custom'
]).decode('utf8').strip()
dockerlaunch_done = dateutils.get_current_time()
dockerlaunch_time = (dockerlaunch_done - grading_began).total_seconds()
grade_items_logging.log_message(is_batch_job, which_untrusted,
submission_path, "dcct:",
dockerlaunch_time,
"docker container created")
# --------------------------------------------------------------------
# COMPILE THE SUBMITTED CODE
with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f:
print("====================================\nCOMPILATION STARTS",
file=f)
# copy submitted files to the tmp compilation directory
tmp_compilation = os.path.join(tmp, "TMP_COMPILATION")
os.mkdir(tmp_compilation)
os.chdir(tmp_compilation)
submission_path = os.path.join(tmp_submission, "submission")
checkout_path = os.path.join(tmp_submission, "checkout")
provided_code_path = os.path.join(tmp_autograding, "provided_code")
test_input_path = os.path.join(tmp_autograding, "test_input")
test_output_path = os.path.join(tmp_autograding, "test_output")
custom_validation_code_path = os.path.join(tmp_autograding,
"custom_validation_code")
bin_path = os.path.join(tmp_autograding, "bin")
form_json_config = os.path.join(tmp_autograding, "form.json")
complete_config = os.path.join(tmp_autograding, "complete_config.json")
with open(form_json_config, 'r') as infile:
gradeable_config_obj = json.load(infile)
gradeable_deadline_string = gradeable_config_obj["date_due"]
with open(complete_config, 'r') as infile:
complete_config_obj = json.load(infile)
patterns_submission_to_compilation = complete_config_obj["autograding"][
"submission_to_compilation"]
pattern_copy("submission_to_compilation",
patterns_submission_to_compilation, submission_path,
tmp_compilation, tmp_logs)
is_vcs = gradeable_config_obj["upload_type"] == "repository"
checkout_subdirectory = complete_config_obj["autograding"].get(
"use_checkout_subdirectory", "")
checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory)
if is_vcs:
pattern_copy("checkout_to_compilation",
patterns_submission_to_compilation, checkout_subdir_path,
tmp_compilation, tmp_logs)
# copy any instructor provided code files to tmp compilation directory
copy_contents_into(provided_code_path, tmp_compilation, tmp_logs)
subprocess.call(['ls', '-lR', '.'],
stdout=open(tmp_logs + "/overall.txt", 'a'))
# copy compile.out to the current directory
shutil.copy(os.path.join(bin_path, "compile.out"),
os.path.join(tmp_compilation, "my_compile.out"))
# give the untrusted user read/write/execute permissions on the tmp directory & files
add_permissions_recursive(tmp_compilation,
stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP,
stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP,
stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP)
add_permissions(tmp, stat.S_IROTH | stat.S_IXOTH)
add_permissions(tmp_logs, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
# grab the submission time
with open(os.path.join(submission_path, ".submit.timestamp"),
'r') as submission_time_file:
submission_string = submission_time_file.read().rstrip()
with open(os.path.join(tmp_logs, "compilation_log.txt"), 'w') as logfile:
if USE_DOCKER:
compile_success = subprocess.call([
'docker', 'exec', '-w', tmp_compilation, container,
os.path.join(tmp_compilation, 'my_compile.out'),
queue_obj['gradeable'], queue_obj['who'],
str(queue_obj['version']), submission_string
],
stdout=logfile)
else:
compile_success = subprocess.call([
os.path.join(SUBMITTY_INSTALL_DIR, "bin",
"untrusted_execute"), which_untrusted,
os.path.join(tmp_compilation, "my_compile.out"),
queue_obj["gradeable"], queue_obj["who"],
str(queue_obj["version"]), submission_string
],
stdout=logfile)
if compile_success == 0:
print("pid", os.getpid(), "COMPILATION OK")
else:
print("pid", os.getpid(), "COMPILATION FAILURE")
grade_items_logging.log_message(is_batch_job,
which_untrusted,
item_name,
message="COMPILATION FAILURE")
untrusted_grant_rwx_access(which_untrusted, tmp_compilation)
# remove the compilation program
os.remove(os.path.join(tmp_compilation, "my_compile.out"))
# return to the main tmp directory
os.chdir(tmp)
# --------------------------------------------------------------------
# make the runner directory
with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f:
print("====================================\nRUNNER STARTS", file=f)
tmp_work = os.path.join(tmp, "TMP_WORK")
os.makedirs(tmp_work)
os.chdir(tmp_work)
# move all executable files from the compilation directory to the main tmp directory
# Note: Must preserve the directory structure of compiled files (esp for Java)
patterns_submission_to_runner = complete_config_obj["autograding"][
"submission_to_runner"]
pattern_copy("submission_to_runner", patterns_submission_to_runner,
submission_path, tmp_work, tmp_logs)
if is_vcs:
pattern_copy("checkout_to_runner", patterns_submission_to_runner,
checkout_subdir_path, tmp_work, tmp_logs)
patterns_compilation_to_runner = complete_config_obj["autograding"][
"compilation_to_runner"]
pattern_copy("compilation_to_runner", patterns_compilation_to_runner,
tmp_compilation, tmp_work, tmp_logs)
# copy input files to tmp_work directory
copy_contents_into(test_input_path, tmp_work, tmp_logs)
subprocess.call(['ls', '-lR', '.'],
stdout=open(tmp_logs + "/overall.txt", 'a'))
# copy runner.out to the current directory
shutil.copy(os.path.join(bin_path, "run.out"),
os.path.join(tmp_work, "my_runner.out"))
# give the untrusted user read/write/execute permissions on the tmp directory & files
add_permissions_recursive(tmp_work,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)
# run the run.out as the untrusted user
with open(os.path.join(tmp_logs, "runner_log.txt"), 'w') as logfile:
print("LOGGING BEGIN my_runner.out", file=logfile)
logfile.flush()
try:
if USE_DOCKER:
runner_success = subprocess.call([
'docker', 'exec', '-w', tmp_work, container,
os.path.join(tmp_work, 'my_runner.out'),
queue_obj['gradeable'], queue_obj['who'],
str(queue_obj['version']), submission_string
],
stdout=logfile)
else:
runner_success = subprocess.call([
os.path.join(SUBMITTY_INSTALL_DIR, "bin",
"untrusted_execute"), which_untrusted,
os.path.join(tmp_work, "my_runner.out"),
queue_obj["gradeable"], queue_obj["who"],
str(queue_obj["version"]), submission_string
],
stdout=logfile)
logfile.flush()
except Exception as e:
print("ERROR caught runner.out exception={0}".format(str(
e.args[0])).encode("utf-8"),
file=logfile)
logfile.flush()
print("LOGGING END my_runner.out", file=logfile)
logfile.flush()
killall_success = subprocess.call([
os.path.join(SUBMITTY_INSTALL_DIR, "bin", "untrusted_execute"),
which_untrusted,
os.path.join(SUBMITTY_INSTALL_DIR, "bin", "killall.py")
],
stdout=logfile)
print("KILLALL COMPLETE my_runner.out", file=logfile)
logfile.flush()
if killall_success != 0:
msg = 'RUNNER ERROR: had to kill {} process(es)'.format(
killall_success)
print("pid", os.getpid(), msg)
grade_items_logging.log_message(is_batch_job, which_untrusted,
item_name, "", "", msg)
if runner_success == 0:
print("pid", os.getpid(), "RUNNER OK")
else:
print("pid", os.getpid(), "RUNNER FAILURE")
grade_items_logging.log_message(is_batch_job,
which_untrusted,
item_name,
message="RUNNER FAILURE")
untrusted_grant_rwx_access(which_untrusted, tmp_work)
untrusted_grant_rwx_access(which_untrusted, tmp_compilation)
# --------------------------------------------------------------------
# RUN VALIDATOR
with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f:
print("====================================\nVALIDATION STARTS",
file=f)
# copy results files from compilation...
patterns_submission_to_validation = complete_config_obj["autograding"][
"submission_to_validation"]
pattern_copy("submission_to_validation", patterns_submission_to_validation,
submission_path, tmp_work, tmp_logs)
if is_vcs:
pattern_copy("checkout_to_validation",
patterns_submission_to_validation, checkout_subdir_path,
tmp_work, tmp_logs)
patterns_compilation_to_validation = complete_config_obj["autograding"][
"compilation_to_validation"]
pattern_copy("compilation_to_validation",
patterns_compilation_to_validation, tmp_compilation, tmp_work,
tmp_logs)
# remove the compilation directory
shutil.rmtree(tmp_compilation)
# copy output files to tmp_work directory
copy_contents_into(test_output_path, tmp_work, tmp_logs)
# copy any instructor custom validation code into the tmp work directory
copy_contents_into(custom_validation_code_path, tmp_work, tmp_logs)
subprocess.call(['ls', '-lR', '.'],
stdout=open(tmp_logs + "/overall.txt", 'a'))
# copy validator.out to the current directory
shutil.copy(os.path.join(bin_path, "validate.out"),
os.path.join(tmp_work, "my_validator.out"))
# give the untrusted user read/write/execute permissions on the tmp directory & files
add_permissions_recursive(tmp_work,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)
add_permissions(os.path.join(tmp_work, "my_validator.out"),
stat.S_IROTH | stat.S_IXOTH)
# validator the validator.out as the untrusted user
with open(os.path.join(tmp_logs, "validator_log.txt"), 'w') as logfile:
if USE_DOCKER:
validator_success = subprocess.call([
'docker', 'exec', '-w', tmp_work, container,
os.path.join(tmp_work, 'my_validator.out'),
queue_obj['gradeable'], queue_obj['who'],
str(queue_obj['version']), submission_string
],
stdout=logfile)
else:
validator_success = subprocess.call([
os.path.join(SUBMITTY_INSTALL_DIR, "bin",
"untrusted_execute"), which_untrusted,
os.path.join(tmp_work, "my_validator.out"),
queue_obj["gradeable"], queue_obj["who"],
str(queue_obj["version"]), submission_string
],
stdout=logfile)
if validator_success == 0:
print("pid", os.getpid(), "VALIDATOR OK")
else:
print("pid", os.getpid(), "VALIDATOR FAILURE")
grade_items_logging.log_message(is_batch_job,
which_untrusted,
item_name,
message="VALIDATION FAILURE")
untrusted_grant_rwx_access(which_untrusted, tmp_work)
# grab the result of autograding
grade_result = ""
with open(os.path.join(tmp_work, "grade.txt")) as f:
lines = f.readlines()
for line in lines:
line = line.rstrip('\n')
if line.startswith("Automatic grading total:"):
grade_result = line
# --------------------------------------------------------------------
# MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE
tmp_results = os.path.join(tmp, "TMP_RESULTS")
with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f:
print("====================================\nARCHIVING STARTS", file=f)
subprocess.call(['ls', '-lR', '.'],
stdout=open(tmp_logs + "/overall.txt", 'a'))
os.makedirs(os.path.join(tmp_results, "details"))
patterns_work_to_details = complete_config_obj["autograding"][
"work_to_details"]
pattern_copy("work_to_details", patterns_work_to_details, tmp_work,
os.path.join(tmp_results, "details"), tmp_logs)
history_file_tmp = os.path.join(tmp_submission, "history.json")
history_file = os.path.join(tmp_results, "history.json")
if os.path.isfile(history_file_tmp):
shutil.move(history_file_tmp, history_file)
# fix permissions
ta_group_id = os.stat(tmp_results).st_gid
os.chown(history_file, int(HWCRON_UID), ta_group_id)
add_permissions(history_file, stat.S_IRGRP)
grading_finished = dateutils.get_current_time()
shutil.copy(os.path.join(tmp_work, "results.json"), tmp_results)
shutil.copy(os.path.join(tmp_work, "grade.txt"), tmp_results)
# -------------------------------------------------------------
# create/append to the results history
# grab the submission time
with open(os.path.join(submission_path,
".submit.timestamp")) as submission_time_file:
submission_string = submission_time_file.read().rstrip()
submission_datetime = dateutils.read_submitty_date(submission_string)
gradeable_deadline_datetime = dateutils.read_submitty_date(
gradeable_deadline_string)
gradeable_deadline_longstring = dateutils.write_submitty_date(
gradeable_deadline_datetime)
submission_longstring = dateutils.write_submitty_date(submission_datetime)
seconds_late = int(
(submission_datetime - gradeable_deadline_datetime).total_seconds())
# note: negative = not late
with open(os.path.join(tmp_submission, ".grading_began"), 'r') as f:
grading_began_longstring = f.read()
grading_began = dateutils.read_submitty_date(grading_began_longstring)
grading_finished_longstring = dateutils.write_submitty_date(
grading_finished)
gradingtime = (grading_finished - grading_began).total_seconds()
with open(os.path.join(tmp_submission, "queue_file.json"), 'r') as infile:
queue_obj = json.load(infile)
queue_obj["gradingtime"] = gradingtime
queue_obj["grade_result"] = grade_result
queue_obj["which_untrusted"] = which_untrusted
with open(os.path.join(tmp_results, "queue_file.json"), 'w') as outfile:
json.dump(queue_obj,
outfile,
sort_keys=True,
indent=4,
separators=(',', ': '))
write_grade_history.just_write_grade_history(
history_file, gradeable_deadline_longstring, submission_longstring,
seconds_late, queue_time_longstring, is_batch_job_string,
grading_began_longstring, int(waittime), grading_finished_longstring,
int(gradingtime), grade_result)
os.chdir(SUBMITTY_DATA_DIR)
if USE_DOCKER:
with open(os.path.join(tmp_logs, "overall_log.txt"), 'w') as logfile:
chmod_success = subprocess.call([
'docker', 'exec', '-w', tmp_work, container, 'chmod', '-R',
'o+rwx', '.'
],
stdout=logfile)
with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f:
f.write("FINISHED GRADING!\n")
# save the logs!
shutil.copytree(tmp_logs, os.path.join(tmp_results, "logs"))
# zip up results folder
my_results_zip_file = tempfile.mkstemp()[1]
zip_my_directory(tmp_results, my_results_zip_file)
shutil.rmtree(tmp_autograding)
shutil.rmtree(tmp_submission)
shutil.rmtree(tmp_results)
shutil.rmtree(tmp_work)
shutil.rmtree(tmp)
# WIP: extra logging for testing
#grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,message="done grading")
# --------------------------------------------------------------------
# CLEAN UP DOCKER
if USE_DOCKER:
subprocess.call(['docker', 'rm', '-f', container])
dockerdestroy_done = dateutils.get_current_time()
dockerdestroy_time = (dockerdestroy_done -
grading_finished).total_seconds()
grade_items_logging.log_message(is_batch_job, which_untrusted,
submission_path, "ddt:",
dockerdestroy_time,
"docker container destroyed")
grade_items_logging.log_message(is_batch_job, which_untrusted, item_name,
"grade:", gradingtime, grade_result)
return my_results_zip_file
def just_grade_item(next_directory,next_to_grade,which_untrusted):
my_pid = os.getpid()
# verify the hwcron user is running this script
if not int(os.getuid()) == int(HWCRON_UID):
grade_items_logging.log_message("ERROR: must be run by hwcron")
raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user")
# --------------------------------------------------------
# figure out what we're supposed to grade & error checking
obj = get_submission_path(next_directory,next_to_grade)
submission_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],
"submissions",obj["gradeable"],obj["who"],str(obj["version"]))
if not os.path.isdir(submission_path):
grade_items_logging.log_message("ERROR: the submission directory does not exist" + submission_path)
raise SystemExit("ERROR: the submission directory does not exist",submission_path)
print ("pid",my_pid,"GRADE THIS", submission_path)
is_vcs,vcs_type,vcs_base_url,vcs_subdirectory = get_vcs_info(SUBMITTY_DATA_DIR,obj["semester"],obj["course"],obj["gradeable"],obj["who"],obj["team"])
is_batch_job = next_directory==BATCH_QUEUE
is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE"
queue_time = get_queue_time(next_directory,next_to_grade)
queue_time_longstring = dateutils.write_submitty_date(queue_time)
grading_began = dateutils.get_current_time()
waittime = int((grading_began-queue_time).total_seconds())
grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"wait:",waittime,"")
# --------------------------------------------------------
# various paths
provided_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"provided_code",obj["gradeable"])
test_input_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_input",obj["gradeable"])
test_output_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"test_output",obj["gradeable"])
custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"custom_validation_code",obj["gradeable"])
bin_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"bin")
checkout_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"checkout",obj["gradeable"],obj["who"],str(obj["version"]))
results_path = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"results",obj["gradeable"],obj["who"],str(obj["version"]))
# grab a copy of the current history.json file (if it exists)
history_file = os.path.join(results_path,"history.json")
history_file_tmp = ""
if os.path.isfile(history_file):
filehandle,history_file_tmp = tempfile.mkstemp()
shutil.copy(history_file,history_file_tmp)
# get info from the gradeable config file
json_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","form","form_"+obj["gradeable"]+".json")
with open(json_config, 'r') as infile:
gradeable_config_obj = json.load(infile)
# get info from the gradeable config file
complete_config = os.path.join(SUBMITTY_DATA_DIR,"courses",obj["semester"],obj["course"],"config","complete_config","complete_config_"+obj["gradeable"]+".json")
with open(complete_config, 'r') as infile:
complete_config_obj = json.load(infile)
checkout_subdirectory = complete_config_obj["autograding"].get("use_checkout_subdirectory","")
checkout_subdir_path = os.path.join(checkout_path,checkout_subdirectory)
# --------------------------------------------------------------------
# MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE
tmp = os.path.join("/var/local/submitty/autograding_tmp/",which_untrusted,"tmp")
shutil.rmtree(tmp,ignore_errors=True)
os.makedirs(tmp)
# switch to tmp directory
os.chdir(tmp)
# make the logs directory
tmp_logs = os.path.join(tmp,"tmp_logs")
os.makedirs(tmp_logs)
# grab the submission time
with open (os.path.join(submission_path,".submit.timestamp")) as submission_time_file:
submission_string = submission_time_file.read().rstrip()
submission_datetime = dateutils.read_submitty_date(submission_string)
# --------------------------------------------------------------------
# CHECKOUT THE STUDENT's REPO
if is_vcs:
with open(os.path.join(tmp_logs,"overall.txt"),'a') as f:
print ("====================================\nVCS CHECKOUT", file=f)
print ("vcs_subdirectory",vcs_subdirectory, file=f)
# cleanup the previous checkout (if it exists)
shutil.rmtree(checkout_path,ignore_errors=True)
os.makedirs(checkout_path, exist_ok=True)
subprocess.call (['/usr/bin/git', 'clone', vcs_subdirectory, checkout_path])
os.chdir(checkout_path)
# determine which version we need to checkout
what_version = subprocess.check_output(['git', 'rev-list', '-n', '1', '--before="'+submission_string+'"', 'master'])
what_version = str(what_version.decode('utf-8')).rstrip()
if what_version == "":
# oops, pressed the grade button before a valid commit
shutil.rmtree(checkout_path,ignore_errors=True)
else:
# and check out the right version
subprocess.call (['git', 'checkout', '-b', 'grade', what_version])
os.chdir(tmp)
subprocess.call(['ls', '-lR', checkout_path], stdout=open(tmp_logs + "/overall.txt", 'a'))
# --------------------------------------------------------------------
# START DOCKER
container = None
if USE_DOCKER:
container = subprocess.check_output(['docker', 'run', '-t', '-d',
'-v', tmp + ':' + tmp,
'ubuntu:custom']).decode('utf8').strip()
# --------------------------------------------------------------------
# COMPILE THE SUBMITTED CODE
with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f:
print("====================================\nCOMPILATION STARTS", file=f)
# copy submitted files to the tmp compilation directory
tmp_compilation = os.path.join(tmp,"TMP_COMPILATION")
os.mkdir(tmp_compilation)
os.chdir(tmp_compilation)
gradeable_deadline_string = gradeable_config_obj["date_due"]
patterns_submission_to_compilation = complete_config_obj["autograding"]["submission_to_compilation"]
pattern_copy("submission_to_compilation",patterns_submission_to_compilation,submission_path,tmp_compilation,tmp_logs)
if is_vcs:
pattern_copy("checkout_to_compilation",patterns_submission_to_compilation,checkout_subdir_path,tmp_compilation,tmp_logs)
# copy any instructor provided code files to tmp compilation directory
copy_contents_into(provided_code_path,tmp_compilation,tmp_logs)
subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a'))
# copy compile.out to the current directory
shutil.copy (os.path.join(bin_path,obj["gradeable"],"compile.out"),os.path.join(tmp_compilation,"my_compile.out"))
# give the untrusted user read/write/execute permissions on the tmp directory & files
add_permissions_recursive(tmp_compilation,
stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP,
stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP,
stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP)
add_permissions(tmp,stat.S_IROTH | stat.S_IXOTH)
add_permissions(tmp_logs,stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
with open(os.path.join(tmp_logs,"compilation_log.txt"), 'w') as logfile:
if USE_DOCKER:
compile_success = subprocess.call(['docker', 'exec', '-w', tmp_compilation, container,
os.path.join(tmp_compilation, 'my_compile.out'), obj['gradeable'],
obj['who'], str(obj['version']), submission_string], stdout=logfile)
else:
compile_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"),
which_untrusted,
os.path.join(tmp_compilation,"my_compile.out"),
obj["gradeable"],
obj["who"],
str(obj["version"]),
submission_string],
stdout=logfile)
if compile_success == 0:
print ("pid",my_pid,"COMPILATION OK")
else:
print ("pid",my_pid,"COMPILATION FAILURE")
grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","COMPILATION FAILURE")
#raise SystemExit()
untrusted_grant_rwx_access(which_untrusted,tmp_compilation)
# remove the compilation program
os.remove(os.path.join(tmp_compilation,"my_compile.out"))
# return to the main tmp directory
os.chdir(tmp)
# --------------------------------------------------------------------
# make the runner directory
with open(os.path.join(tmp_logs,"overall.txt"),'a') as f:
print ("====================================\nRUNNER STARTS", file=f)
tmp_work = os.path.join(tmp,"TMP_WORK")
os.makedirs(tmp_work)
os.chdir(tmp_work)
# move all executable files from the compilation directory to the main tmp directory
# Note: Must preserve the directory structure of compiled files (esp for Java)
patterns_submission_to_runner = complete_config_obj["autograding"]["submission_to_runner"]
pattern_copy("submission_to_runner",patterns_submission_to_runner,submission_path,tmp_work,tmp_logs)
if is_vcs:
pattern_copy("checkout_to_runner",patterns_submission_to_runner,checkout_subdir_path,tmp_work,tmp_logs)
patterns_compilation_to_runner = complete_config_obj["autograding"]["compilation_to_runner"]
pattern_copy("compilation_to_runner",patterns_compilation_to_runner,tmp_compilation,tmp_work,tmp_logs)
# copy input files to tmp_work directory
copy_contents_into(test_input_path,tmp_work,tmp_logs)
subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a'))
# copy runner.out to the current directory
shutil.copy (os.path.join(bin_path,obj["gradeable"],"run.out"),os.path.join(tmp_work,"my_runner.out"))
# give the untrusted user read/write/execute permissions on the tmp directory & files
add_permissions_recursive(tmp_work,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)
# raise SystemExit()
# run the run.out as the untrusted user
with open(os.path.join(tmp_logs,"runner_log.txt"), 'w') as logfile:
if USE_DOCKER:
runner_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container,
os.path.join(tmp_work, 'my_runner.out'), obj['gradeable'],
obj['who'], str(obj['version']), submission_string], stdout=logfile)
else:
runner_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"),
which_untrusted,
os.path.join(tmp_work,"my_runner.out"),
obj["gradeable"],
obj["who"],
str(obj["version"]),
submission_string],
stdout=logfile)
if runner_success == 0:
print ("pid",my_pid,"RUNNER OK")
else:
print ("pid",my_pid,"RUNNER FAILURE")
grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","RUNNER FAILURE")
untrusted_grant_rwx_access(which_untrusted,tmp_work)
untrusted_grant_rwx_access(which_untrusted,tmp_compilation)
# --------------------------------------------------------------------
# RUN VALIDATOR
with open(os.path.join(tmp_logs,"overall.txt"),'a') as f:
print ("====================================\nVALIDATION STARTS", file=f)
# copy results files from compilation...
patterns_submission_to_validation = complete_config_obj["autograding"]["submission_to_validation"]
pattern_copy("submission_to_validation",patterns_submission_to_validation,submission_path,tmp_work,tmp_logs)
if is_vcs:
pattern_copy("checkout_to_validation",patterns_submission_to_validation,checkout_subdir_path,tmp_work,tmp_logs)
patterns_compilation_to_validation = complete_config_obj["autograding"]["compilation_to_validation"]
pattern_copy("compilation_to_validation",patterns_compilation_to_validation,tmp_compilation,tmp_work,tmp_logs)
# remove the compilation directory
shutil.rmtree(tmp_compilation)
# copy output files to tmp_work directory
copy_contents_into(test_output_path,tmp_work,tmp_logs)
# copy any instructor custom validation code into the tmp work directory
copy_contents_into(custom_validation_code_path,tmp_work,tmp_logs)
subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a'))
# copy validator.out to the current directory
shutil.copy (os.path.join(bin_path,obj["gradeable"],"validate.out"),os.path.join(tmp_work,"my_validator.out"))
# give the untrusted user read/write/execute permissions on the tmp directory & files
add_permissions_recursive(tmp_work,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH,
stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)
add_permissions(os.path.join(tmp_work,"my_validator.out"),stat.S_IROTH | stat.S_IXOTH)
# validator the validator.out as the untrusted user
with open(os.path.join(tmp_logs,"validator_log.txt"), 'w') as logfile:
if USE_DOCKER:
validator_success = subprocess.call(['docker', 'exec', '-w', tmp_work, container,
os.path.join(tmp_work, 'my_validator.out'), obj['gradeable'],
obj['who'], str(obj['version']), submission_string], stdout=logfile)
else:
validator_success = subprocess.call([os.path.join(SUBMITTY_INSTALL_DIR,"bin","untrusted_execute"),
which_untrusted,
os.path.join(tmp_work,"my_validator.out"),
obj["gradeable"],
obj["who"],
str(obj["version"]),
submission_string],
stdout=logfile)
if validator_success == 0:
print ("pid",my_pid,"VALIDATOR OK")
else:
print ("pid",my_pid,"VALIDATOR FAILURE")
grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"","","VALIDATION FAILURE")
untrusted_grant_rwx_access(which_untrusted,tmp_work)
# grab the result of autograding
grade_result = ""
with open(os.path.join(tmp_work,"grade.txt")) as f:
lines = f.readlines()
for line in lines:
line = line.rstrip('\n')
if line.startswith("Automatic grading total:"):
grade_result = line
# --------------------------------------------------------------------
# MAKE RESULTS DIRECTORY & COPY ALL THE FILES THERE
with open(os.path.join(tmp_logs,"overall.txt"),'a') as f:
print ("====================================\nARCHIVING STARTS", file=f)
subprocess.call(['ls', '-lR', '.'], stdout=open(tmp_logs + "/overall.txt", 'a'))
os.chdir(bin_path)
# save the old results path!
if os.path.isdir(os.path.join(results_path,"OLD")):
shutil.move(os.path.join(results_path,"OLD"),
os.path.join(tmp,"OLD_RESULTS"))
# clean out all of the old files if this is a re-run
shutil.rmtree(results_path,ignore_errors=True)
# create the directory (and the full path if it doesn't already exist)
os.makedirs(results_path)
# bring back the old results!
if os.path.isdir(os.path.join(tmp,"OLD_RESULTS")):
shutil.move(os.path.join(tmp,"OLD_RESULTS"),
os.path.join(results_path,"OLD"))
os.makedirs(os.path.join(results_path,"details"))
patterns_work_to_details = complete_config_obj["autograding"]["work_to_details"]
pattern_copy("work_to_details",patterns_work_to_details,tmp_work,os.path.join(results_path,"details"),tmp_logs)
if not history_file_tmp == "":
shutil.move(history_file_tmp,history_file)
# fix permissions
ta_group_id = os.stat(results_path).st_gid
os.chown(history_file,int(HWCRON_UID),ta_group_id)
add_permissions(history_file,stat.S_IRGRP)
grading_finished = dateutils.get_current_time()
shutil.copy(os.path.join(tmp_work,"results.json"),results_path)
shutil.copy(os.path.join(tmp_work,"grade.txt"),results_path)
# -------------------------------------------------------------
# create/append to the results history
gradeable_deadline_datetime = dateutils.read_submitty_date(gradeable_deadline_string)
gradeable_deadline_longstring = dateutils.write_submitty_date(gradeable_deadline_datetime)
submission_longstring = dateutils.write_submitty_date(submission_datetime)
seconds_late = int((submission_datetime-gradeable_deadline_datetime).total_seconds())
# note: negative = not late
grading_began_longstring = dateutils.write_submitty_date(grading_began)
grading_finished_longstring = dateutils.write_submitty_date(grading_finished)
gradingtime = int((grading_finished-grading_began).total_seconds())
write_grade_history.just_write_grade_history(history_file,
gradeable_deadline_longstring,
submission_longstring,
seconds_late,
queue_time_longstring,
is_batch_job_string,
grading_began_longstring,
waittime,
grading_finished_longstring,
gradingtime,
grade_result)
#---------------------------------------------------------------------
# WRITE OUT VERSION DETAILS
if WRITE_DATABASE:
insert_database_version_data.insert_to_database(
obj["semester"],
obj["course"],
obj["gradeable"],
obj["user"],
obj["team"],
obj["who"],
True if obj["is_team"] else False,
str(obj["version"]))
print ("pid",my_pid,"finished grading ", next_to_grade, " in ", gradingtime, " seconds")
grade_items_logging.log_message(is_batch_job,which_untrusted,submission_path,"grade:",gradingtime,grade_result)
with open(os.path.join(tmp_logs,"overall.txt"),'a') as f:
f.write("FINISHED GRADING!")
# save the logs!
shutil.copytree(tmp_logs,os.path.join(results_path,"logs"))
# --------------------------------------------------------------------
# REMOVE TEMP DIRECTORY
shutil.rmtree(tmp)
# --------------------------------------------------------------------
# CLEAN UP DOCKER
if USE_DOCKER:
subprocess.call(['docker', 'rm', '-f', container])
def launch_workers(num_workers):
# verify the hwcron user is running this script
if not int(os.getuid()) == int(HWCRON_UID):
raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user")
grade_items_logging.log_message(message="grade_scheduler.py launched")
# prepare a list of untrusted users to be used by the workers
untrusted_users = multiprocessing.Queue()
for i in range(num_workers):
untrusted_users.put("untrusted" + str(i).zfill(2))
# Set up our queues that we're going to monitor for new jobs to run on
interactive_queue = multiprocessing.Queue()
batch_queue = multiprocessing.Queue()
populate_queue(interactive_queue, INTERACTIVE_QUEUE)
populate_queue(batch_queue, BATCH_QUEUE)
# the workers will wait on event if the queues are exhausted
new_job_event = multiprocessing.Event()
# this lock will be used to edit the queue or new job event
overall_lock = multiprocessing.Lock()
# Setup watchdog observer that will watch the folders and run the handler on any
# FileSystemEvents. This runs in a thread automatically.
interactive_handler = NewFileHandler(interactive_queue,new_job_event,overall_lock)
batch_handler = NewFileHandler(batch_queue,new_job_event,overall_lock)
observer = Observer()
observer.schedule(event_handler=interactive_handler, path=INTERACTIVE_QUEUE, recursive=False)
observer.schedule(event_handler=batch_handler, path=BATCH_QUEUE, recursive=False)
observer.start()
# launch the worker threads
processes = list()
for i in range(0,num_workers):
u = "untrusted" + str(i).zfill(2)
p = multiprocessing.Process(target=worker_process,args=(interactive_queue,batch_queue,new_job_event,overall_lock,u))
p.start()
processes.append(p)
# main monitoring loop
try:
while True:
alive = 0
for i in range(0,num_workers):
if processes[i].is_alive:
alive = alive+1
else:
grade_items_logging.log_message(message="ERROR: process "+str(i)+" is not alive")
if alive != num_workers:
grade_items_logging.log_message(message="ERROR: #workers="+str(num_workers)+" != #alive="+str(alive))
#print ("workers= ",num_workers," alive=",alive)
time.sleep(1)
except KeyboardInterrupt:
grade_items_logging.log_message(message="grade_scheduler.py keyboard interrupt")
# just kill everything in this group id right now
# NOTE: this may be a bug if the grandchildren have a different group id and not be killed
os.kill(-os.getpid(), signal.SIGKILL)
# run this to check if everything is dead
# ps xao pid,ppid,pgid,sid,comm,user | grep untrust
# everything's dead, including the main process so the rest of this will be ignored
# but this was mostly working...
# terminate the jobs
for i in range(0,num_workers):
processes[i].terminate()
# wake up sleeping jobs
new_job_event.set()
# wait for them to join
for i in range(0,num_workers):
processes[i].join()
# cleanup observer
observer.stop()
observer.join()
grade_items_logging.log_message(message="grade_scheduler.py terminated")
def update_foreign_autograding_worker_json(name, entry):
fd, tmp_json_path = tempfile.mkstemp()
foreign_json = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO","autograding_worker.json")
autograding_worker_to_ship = entry
try:
user = autograding_worker_to_ship[name]['username']
host = autograding_worker_to_ship[name]['address']
except Exception as e:
print("ERROR: autograding_workers.json entry for {0} is malformatted. {1}".format(e, name))
grade_items_logging.log_message(JOB_ID, message="ERROR: autograding_workers.json entry for {0} is malformatted. {1}".format(e, name))
return
#create a new temporary json with only the entry for the current machine.
with open(tmp_json_path, 'w') as outfile:
json.dump(autograding_worker_to_ship, outfile, sort_keys=True, indent=4)
#if we are updating the current machine, we can just move the new json to the appropriate spot (no ssh needed)
if host == "localhost":
try:
shutil.move(tmp_json_path,foreign_json)
print("Successfully updated local autograding_TODO/autograding_worker.json")
grade_items_logging.log_message(JOB_ID, message="Successfully updated local autograding_TODO/autograding_worker.json")
except Exception as e:
grade_items_logging.log_message(JOB_ID, message="ERROR: could not mv to local autograding_TODO/autograding_worker.json due to the following error: "+str(e))
print("ERROR: could not mv to local autograding_worker.json due to the following error: {0}".format(e))
finally:
os.close(fd)
#if we are updating a foreign machine, we must connect via ssh and use sftp to update it.
else:
#try to establish an ssh connection to the host
try:
ssh = paramiko.SSHClient()
ssh.get_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname = host, username = user)
except Exception as e:
grade_items_logging.log_message(JOB_ID, message="ERROR: could not ssh to "+host+" due to following error: "+str(e))
print("ERROR: could not ssh to "+host+" due to following error: "+str(e))
return
#try to copy the files over to the host
try:
sftp = ssh.open_sftp()
sftp.put(tmp_json_path,foreign_json)
sftp.close()
print("Successfully forwarded autograding_worker.json to {0}".format(name))
grade_items_logging.log_message(JOB_ID, message="Successfully forwarded autograding_worker.json to {0}".format(name))
except Exception as e:
grade_items_logging.log_message(JOB_ID, message="ERROR: could not sftp to foreign autograding_TODO/autograding_worker.json due to the following error: "+str(e))
print("ERROR: could sftp to foreign autograding_TODO/autograding_worker.json due to the following error: {0}".format(e))
success = False
finally:
os.close(fd)
os.remove(tmp_json_path)
sftp.close()
ssh.close()
def launch_shippers():
# verify the hwcron user is running this script
if not int(os.getuid()) == int(HWCRON_UID):
raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user")
grade_items_logging.log_message(JOB_ID, message="grade_scheduler.py launched")
# Clean up old files from previous shipping/autograding (any
# partially completed work will be re-done)
for file_path in glob.glob(os.path.join(INTERACTIVE_QUEUE, "GRADING_*")):
grade_items_logging.log_message(JOB_ID, message="Remove old queue file: " + file_path)
os.remove(file_path)
for file_path in glob.glob(os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO","unstrusted*")):
grade_items_logging.log_message(JOB_ID, message="Remove autograding TODO file: " + file_path)
os.remove(file_path)
for file_path in glob.glob(os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE","*")):
grade_items_logging.log_message(JOB_ID, message="Remove autograding DONE file: " + file_path)
os.remove(file_path)
# this lock will be used to edit the queue or new job event
overall_lock = multiprocessing.Lock()
# The names of the worker machines, the capabilities of each
# worker machine, and the number of workers per machine are stored
# in the autograding_workers json.
try:
autograding_workers_path = os.path.join(SUBMITTY_INSTALL_DIR, 'config', "autograding_workers.json")
with open(autograding_workers_path, 'r') as infile:
autograding_workers = json.load(infile)
except Exception as e:
raise SystemExit("ERROR: could not locate the autograding workers json: {0}".format(e))
# There must always be a primary machine, it may or may not have
# autograding workers.
if not "primary" in autograding_workers:
raise SystemExit("ERROR: autograding_workers.json contained no primary machine.")
# One (or more) of the machines must accept "default" jobs.
default_present = False
for name, machine in autograding_workers.items():
if "default" in machine["capabilities"]:
default_present = True
break
if not default_present:
raise SystemExit("ERROR: autograding_workers.json contained no machine with default capabilities")
# Launch a shipper process for every work on the primary machine and each worker machine
total_num_workers = 0
processes = list()
for name, machine in autograding_workers.items():
try:
which_machine=machine["address"]
if which_machine != "localhost":
if machine["username"] == "":
raise SystemExit("ERROR: empty username for worker machine '" + which_machine + "'")
which_machine = "{0}@{1}".format(machine["username"], machine["address"])
elif not machine["username"] == "":
Raise('ERROR: username for primary (localhost) must be ""')
num_workers_on_machine = machine["num_autograding_workers"]
if num_workers_on_machine < 0:
raise SystemExit("ERROR: num_workers_on_machine for '" + which_machine + "' must be non-negative.")
my_capabilities = machine["capabilities"]
except Exception as e:
print("ERROR: autograding_workers.json entry for {0} contains an error: {1}".format(name, e))
grade_items_logging.log_message(JOB_ID, message="ERROR: autograding_workers.json entry for {0} contains an error: {1}".format(name,e))
continue
# launch the shipper threads
for i in range(0,num_workers_on_machine):
u = "untrusted" + str(i).zfill(2)
p = multiprocessing.Process(target=shipper_process,args=(name,which_machine,my_capabilities,u,overall_lock))
p.start()
processes.append(p)
total_num_workers += num_workers_on_machine
# main monitoring loop
try:
while True:
alive = 0
for i in range(0,total_num_workers):
if processes[i].is_alive:
alive = alive+1
else:
grade_items_logging.log_message(JOB_ID, message="ERROR: process "+str(i)+" is not alive")
if alive != total_num_workers:
grade_items_logging.log_message(JOB_ID, message="ERROR: #shippers="+str(total_num_workers)+" != #alive="+str(alive))
#print ("shippers= ",total_num_workers," alive=",alive)
time.sleep(1)
except KeyboardInterrupt:
grade_items_logging.log_message(JOB_ID, message="grade_scheduler.py keyboard interrupt")
# just kill everything in this group id right now
# NOTE: this may be a bug if the grandchildren have a different group id and not be killed
os.kill(-os.getpid(), signal.SIGKILL)
# run this to check if everything is dead
# ps xao pid,ppid,pgid,sid,comm,user | grep untrust
# everything's dead, including the main process so the rest of this will be ignored
# but this was mostly working...
# terminate the jobs
for i in range(0,total_num_workers):
processes[i].terminate()
# wait for them to join
for i in range(0,total_num_workers):
processes[i].join()
grade_items_logging.log_message(JOB_ID, message="grade_scheduler.py terminated")
def launch_workers(my_name, my_stats):
num_workers = my_stats['num_autograding_workers']
# verify the hwcron user is running this script
if not int(os.getuid()) == int(HWCRON_UID):
raise SystemExit(
"ERROR: the grade_item.py script must be run by the hwcron user")
grade_items_logging.log_message(JOB_ID,
message="grade_scheduler.py launched")
# prepare a list of untrusted users to be used by the workers
untrusted_users = multiprocessing.Queue()
for i in range(num_workers):
untrusted_users.put("untrusted" + str(i).zfill(2))
# launch the worker threads
address = my_stats['address']
if address != 'localhost':
which_machine = "{0}@{1}".format(my_stats['username'], address)
else:
which_machine = address
my_server = my_stats['server_name']
processes = list()
for i in range(0, num_workers):
u = "untrusted" + str(i).zfill(2)
p = multiprocessing.Process(target=worker_process,
args=(which_machine, address, u,
my_server))
p.start()
processes.append(p)
# main monitoring loop
try:
while True:
alive = 0
for i in range(0, num_workers):
if processes[i].is_alive:
alive = alive + 1
else:
grade_items_logging.log_message(JOB_ID,
message="ERROR: process " +
str(i) + " is not alive")
if alive != num_workers:
grade_items_logging.log_message(JOB_ID,
message="ERROR: #workers=" +
str(num_workers) +
" != #alive=" + str(alive))
#print ("workers= ",num_workers," alive=",alive)
time.sleep(1)
except KeyboardInterrupt:
grade_items_logging.log_message(
JOB_ID, message="grade_scheduler.py keyboard interrupt")
# just kill everything in this group id right now
# NOTE: this may be a bug if the grandchildren have a different group id and not be killed
os.kill(-os.getpid(), signal.SIGKILL)
# run this to check if everything is dead
# ps xao pid,ppid,pgid,sid,comm,user | grep untrust
# everything's dead, including the main process so the rest of this will be ignored
# but this was mostly working...
# terminate the jobs
for i in range(0, num_workers):
processes[i].terminate()
# wait for them to join
for i in range(0, num_workers):
processes[i].join()
grade_items_logging.log_message(JOB_ID,
message="grade_scheduler.py terminated")
def prepare_autograding_and_submission_zip(next_directory, next_to_grade):
os.chdir(SUBMITTY_DATA_DIR)
# --------------------------------------------------------
# figure out what we're supposed to grade & error checking
obj = load_queue_file_obj(next_directory, next_to_grade)
partial_path = os.path.join(obj["gradeable"], obj["who"],
str(obj["version"]))
item_name = os.path.join(obj["semester"], obj["course"], "submissions",
partial_path)
submission_path = os.path.join(SUBMITTY_DATA_DIR, "courses", item_name)
if not os.path.isdir(submission_path):
grade_items_logging.log_message(
message="ERROR: the submission directory does not exist" +
submission_path)
raise RuntimeError("ERROR: the submission directory does not exist",
submission_path)
print("pid", os.getpid(), "GRADE THIS", submission_path)
is_vcs, vcs_type, vcs_base_url, vcs_subdirectory = get_vcs_info(
SUBMITTY_DATA_DIR, obj["semester"], obj["course"], obj["gradeable"],
obj["who"], obj["team"])
is_batch_job = next_directory == BATCH_QUEUE
is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE"
queue_time = get_queue_time(next_directory, next_to_grade)
queue_time_longstring = dateutils.write_submitty_date(queue_time)
grading_began = dateutils.get_current_time()
waittime = (grading_began - queue_time).total_seconds()
grade_items_logging.log_message(is_batch_job, "zip", item_name, "wait:",
waittime, "")
# --------------------------------------------------------------------
# MAKE TEMPORARY DIRECTORY & COPY THE NECESSARY FILES THERE
tmp = tempfile.mkdtemp()
tmp_autograding = os.path.join(tmp, "TMP_AUTOGRADING")
os.mkdir(tmp_autograding)
tmp_submission = os.path.join(tmp, "TMP_SUBMISSION")
os.mkdir(tmp_submission)
# --------------------------------------------------------
# various paths
provided_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses",
obj["semester"], obj["course"],
"provided_code", obj["gradeable"])
test_input_path = os.path.join(SUBMITTY_DATA_DIR, "courses",
obj["semester"], obj["course"],
"test_input", obj["gradeable"])
test_output_path = os.path.join(SUBMITTY_DATA_DIR, "courses",
obj["semester"], obj["course"],
"test_output", obj["gradeable"])
custom_validation_code_path = os.path.join(SUBMITTY_DATA_DIR, "courses",
obj["semester"], obj["course"],
"custom_validation_code",
obj["gradeable"])
bin_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"],
obj["course"], "bin", obj["gradeable"])
form_json_config = os.path.join(SUBMITTY_DATA_DIR, "courses",
obj["semester"], obj["course"], "config",
"form",
"form_" + obj["gradeable"] + ".json")
complete_config = os.path.join(
SUBMITTY_DATA_DIR, "courses", obj["semester"], obj["course"], "config",
"complete_config", "complete_config_" + obj["gradeable"] + ".json")
copytree_if_exists(provided_code_path,
os.path.join(tmp_autograding, "provided_code"))
copytree_if_exists(test_input_path,
os.path.join(tmp_autograding, "test_input"))
copytree_if_exists(test_output_path,
os.path.join(tmp_autograding, "test_output"))
copytree_if_exists(custom_validation_code_path,
os.path.join(tmp_autograding, "custom_validation_code"))
copytree_if_exists(bin_path, os.path.join(tmp_autograding, "bin"))
shutil.copy(form_json_config, os.path.join(tmp_autograding, "form.json"))
shutil.copy(complete_config,
os.path.join(tmp_autograding, "complete_config.json"))
checkout_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"],
obj["course"], "checkout", partial_path)
results_path = os.path.join(SUBMITTY_DATA_DIR, "courses", obj["semester"],
obj["course"], "results", partial_path)
# grab a copy of the current history.json file (if it exists)
history_file = os.path.join(results_path, "history.json")
history_file_tmp = ""
if os.path.isfile(history_file):
filehandle, history_file_tmp = tempfile.mkstemp()
shutil.copy(history_file, history_file_tmp)
shutil.copy(history_file, os.path.join(tmp_submission, "history.json"))
# get info from the gradeable config file
with open(complete_config, 'r') as infile:
complete_config_obj = json.load(infile)
checkout_subdirectory = complete_config_obj["autograding"].get(
"use_checkout_subdirectory", "")
checkout_subdir_path = os.path.join(checkout_path, checkout_subdirectory)
queue_file = os.path.join(next_directory, next_to_grade)
# switch to tmp directory
os.chdir(tmp)
# make the logs directory
tmp_logs = os.path.join(tmp, "TMP_SUBMISSION", "tmp_logs")
os.makedirs(tmp_logs)
# 'touch' a file in the logs folder
open(os.path.join(tmp_logs, "overall.txt"), 'a')
# grab the submission time
with open(os.path.join(submission_path,
".submit.timestamp")) as submission_time_file:
submission_string = submission_time_file.read().rstrip()
submission_datetime = dateutils.read_submitty_date(submission_string)
# --------------------------------------------------------------------
# CHECKOUT THE STUDENT's REPO
if is_vcs:
# is vcs_subdirectory standalone or should it be combined with base_url?
if vcs_subdirectory[0] == '/' or '://' in vcs_subdirectory:
vcs_path = vcs_subdirectory
else:
if '://' in vcs_base_url:
vcs_path = urllib.parse.urljoin(vcs_base_url, vcs_subdirectory)
else:
vcs_path = os.path.join(vcs_base_url, vcs_subdirectory)
with open(os.path.join(tmp_logs, "overall.txt"), 'a') as f:
print("====================================\nVCS CHECKOUT", file=f)
print('vcs_base_url', vcs_base_url, file=f)
print('vcs_subdirectory', vcs_subdirectory, file=f)
print('vcs_path', vcs_path, file=f)
print(['/usr/bin/git', 'clone', vcs_path, checkout_path], file=f)
# cleanup the previous checkout (if it exists)
shutil.rmtree(checkout_path, ignore_errors=True)
os.makedirs(checkout_path, exist_ok=True)
subprocess.call(['/usr/bin/git', 'clone', vcs_path, checkout_path])
os.chdir(checkout_path)
# determine which version we need to checkout
what_version = subprocess.check_output([
'git', 'rev-list', '-n', '1',
'--before="' + submission_string + '"', 'master'
])
what_version = str(what_version.decode('utf-8')).rstrip()
if what_version == "":
# oops, pressed the grade button before a valid commit
shutil.rmtree(checkout_path, ignore_errors=True)
else:
# and check out the right version
subprocess.call(['git', 'checkout', '-b', 'grade', what_version])
os.chdir(tmp)
subprocess.call(['ls', '-lR', checkout_path],
stdout=open(tmp_logs + "/overall.txt", 'a'))
copytree_if_exists(submission_path,
os.path.join(tmp_submission, "submission"))
copytree_if_exists(checkout_path, os.path.join(tmp_submission, "checkout"))
obj["queue_time"] = queue_time_longstring
obj["is_batch_job"] = is_batch_job
obj["waittime"] = waittime
with open(os.path.join(tmp_submission, "queue_file.json"), 'w') as outfile:
json.dump(obj,
outfile,
sort_keys=True,
indent=4,
separators=(',', ': '))
grading_began_longstring = dateutils.write_submitty_date(grading_began)
with open(os.path.join(tmp_submission, ".grading_began"), 'w') as f:
print(grading_began_longstring, file=f)
# zip up autograding & submission folders
my_autograding_zip_file = tempfile.mkstemp()[1]
my_submission_zip_file = tempfile.mkstemp()[1]
zip_my_directory(tmp_autograding, my_autograding_zip_file)
zip_my_directory(tmp_submission, my_submission_zip_file)
# cleanup
shutil.rmtree(tmp_autograding)
shutil.rmtree(tmp_submission)
shutil.rmtree(tmp)
return (my_autograding_zip_file, my_submission_zip_file)
def unpack_job(which_machine,which_untrusted,next_directory,next_to_grade):
# verify the hwcron user is running this script
if not int(os.getuid()) == int(HWCRON_UID):
grade_items_logging.log_message(JOB_ID, message="ERROR: must be run by hwcron")
raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user")
if which_machine == 'localhost':
address = which_machine
else:
address = which_machine.split('@')[1]
fully_qualified_domain_name = socket.getfqdn()
servername_workername = "{0}_{1}".format(fully_qualified_domain_name, address)
target_results_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_results.zip")
target_done_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_DONE",servername_workername+"_"+which_untrusted+"_queue.json")
if which_machine == "localhost":
if not os.path.exists(target_done_queue_file):
return False
else:
local_done_queue_file = target_done_queue_file
local_results_zip = target_results_zip
else:
user, host = which_machine.split("@")
ssh = paramiko.SSHClient()
ssh.get_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(hostname = host, username = user)
sftp = ssh.open_sftp()
fd1, local_done_queue_file = tempfile.mkstemp()
fd2, local_results_zip = tempfile.mkstemp()
#remote path first, then local.
sftp.get(target_done_queue_file, local_done_queue_file)
sftp.get(target_results_zip, local_results_zip)
#Because get works like cp rather tnan mv, we have to clean up.
sftp.remove(target_done_queue_file)
sftp.remove(target_results_zip)
success = True
#This is the normal case (still grading on the other end) so we don't need to print anything.
except FileNotFoundError:
os.remove(local_results_zip)
os.remove(local_done_queue_file)
success = False
#In this more general case, we do want to print what the error was.
#TODO catch other types of exception as we identify them.
except Exception as e:
grade_items_logging.log_message(JOB_ID, message="ERROR: Could not retrieve the file from the foreign machine "+str(e))
print("ERROR: Could not retrieve the file from the foreign machine.\nERROR: {0}".format(e))
os.remove(local_results_zip)
os.remove(local_done_queue_file)
success = False
finally:
os.close(fd1)
os.close(fd2)
sftp.close()
ssh.close()
if not success:
return False
# archive the results of grading
try:
grade_item.unpack_grading_results_zip(which_machine,which_untrusted,local_results_zip)
except:
grade_items_logging.log_message(JOB_ID,jobname=next_to_grade,message="ERROR: Exception when unpacking zip")
with contextlib.suppress(FileNotFoundError):
os.remove(local_results_zip)
with contextlib.suppress(FileNotFoundError):
os.remove(local_done_queue_file)
return True
def prepare_job(my_name,which_machine,which_untrusted,next_directory,next_to_grade):
# verify the hwcron user is running this script
if not int(os.getuid()) == int(HWCRON_UID):
grade_items_logging.log_message(JOB_ID, message="ERROR: must be run by hwcron")
raise SystemExit("ERROR: the grade_item.py script must be run by the hwcron user")
if which_machine == 'localhost':
address = which_machine
else:
address = which_machine.split('@')[1]
# prepare the zip files
try:
autograding_zip_tmp,submission_zip_tmp = grade_item.prepare_autograding_and_submission_zip(which_machine,which_untrusted,next_directory,next_to_grade)
fully_qualified_domain_name = socket.getfqdn()
servername_workername = "{0}_{1}".format(fully_qualified_domain_name, address)
autograding_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_autograding.zip")
submission_zip = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_submission.zip")
todo_queue_file = os.path.join(SUBMITTY_DATA_DIR,"autograding_TODO",servername_workername+"_"+which_untrusted+"_queue.json")
with open(next_to_grade, 'r') as infile:
queue_obj = json.load(infile)
queue_obj["which_untrusted"] = which_untrusted
queue_obj["which_machine"] = which_machine
queue_obj["ship_time"] = dateutils.write_submitty_date(microseconds=True)
except Exception as e:
grade_items_logging.log_message(JOB_ID, message="ERROR: failed preparing submission zip or accessing next to grade "+str(e))
print("ERROR: failed preparing submission zip or accessing next to grade ", e)
return False
if address == "localhost":
try:
shutil.move(autograding_zip_tmp,autograding_zip)
shutil.move(submission_zip_tmp,submission_zip)
with open(todo_queue_file, 'w') as outfile:
json.dump(queue_obj, outfile, sort_keys=True, indent=4)
except Exception as e:
grade_items_logging.log_message(JOB_ID, message="ERROR: could not move files due to the following error: "+str(e))
print("ERROR: could not move files due to the following error: {0}".format(e))
return False
else:
try:
user, host = which_machine.split("@")
ssh = paramiko.SSHClient()
ssh.get_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname = host, username = user)
sftp = ssh.open_sftp()
sftp.put(autograding_zip_tmp,autograding_zip)
sftp.put(submission_zip_tmp,submission_zip)
with open(todo_queue_file, 'w') as outfile:
json.dump(queue_obj, outfile, sort_keys=True, indent=4)
sftp.put(todo_queue_file, todo_queue_file)
os.remove(todo_queue_file)
print("Successfully forwarded files to {0}".format(my_name))
success = True
except Exception as e:
grade_items_logging.log_message(JOB_ID, message="ERROR: could not move files due to the following error: "+str(e))
print("Could not move files due to the following error: {0}".format(e))
success = False
finally:
sftp.close()
ssh.close()
os.remove(autograding_zip_tmp)
os.remove(submission_zip_tmp)
return success
return True
def worker_process(which_machine, address, which_untrusted, my_server):
# verify the hwcron user is running this script
if not int(os.getuid()) == int(HWCRON_UID):
grade_items_logging.log_message(JOB_ID,
message="ERROR: must be run by hwcron")
raise SystemExit(
"ERROR: the grade_item.py script must be run by the hwcron user")
# ignore keyboard interrupts in the worker processes
signal.signal(signal.SIGINT, signal.SIG_IGN)
counter = 0
servername_workername = "{0}_{1}".format(my_server, address)
autograding_zip = os.path.join(
SUBMITTY_DATA_DIR, "autograding_TODO",
servername_workername + "_" + which_untrusted + "_autograding.zip")
submission_zip = os.path.join(
SUBMITTY_DATA_DIR, "autograding_TODO",
servername_workername + "_" + which_untrusted + "_submission.zip")
todo_queue_file = os.path.join(
SUBMITTY_DATA_DIR, "autograding_TODO",
servername_workername + "_" + which_untrusted + "_queue.json")
while True:
if os.path.exists(todo_queue_file):
try:
results_zip_tmp = grade_item.grade_from_zip(
autograding_zip, submission_zip, which_untrusted)
results_zip = os.path.join(
SUBMITTY_DATA_DIR, "autograding_DONE",
servername_workername + "_" + which_untrusted +
"_results.zip")
done_queue_file = os.path.join(
SUBMITTY_DATA_DIR, "autograding_DONE",
servername_workername + "_" + which_untrusted +
"_queue.json")
#move doesn't inherit the permissions of the destination directory. Copyfile does.
shutil.copyfile(results_zip_tmp, results_zip)
os.remove(results_zip_tmp)
with open(todo_queue_file, 'r') as infile:
queue_obj = json.load(infile)
queue_obj["done_time"] = dateutils.write_submitty_date(
microseconds=True)
with open(done_queue_file, 'w') as outfile:
json.dump(queue_obj, outfile, sort_keys=True, indent=4)
except Exception as e:
grade_items_logging.log_message(
JOB_ID,
message="ERROR attempting to unzip graded item: " +
which_machine + " " + which_untrusted + " exception " +
repr(e))
with contextlib.suppress(FileNotFoundError):
os.remove(autograding_zip)
with contextlib.suppress(FileNotFoundError):
os.remove(submission_zip)
with contextlib.suppress(FileNotFoundError):
os.remove(todo_queue_file)
counter = 0
else:
if counter >= 10:
print(which_machine, which_untrusted, "wait")
counter = 0
counter += 1
time.sleep(1)
