def match_found(flock_manager, sheep_identity, request):
logger.info(
"Sending test request for submission [%s] to sheep [%s].",
request.submission_id,
repr(sheep_identity)
)
# Get submission and test harness to send to sheep
submission = Submission.objects(id = request.submission_id).exclude(
"most_recent",
"uploaded_filenames"
).first()
assignment = Assignment.objects.get(id = submission.assignment)
test_harness = TestHarness.objects.get(id = assignment.test_harness)
# Apply any personal deadlines to the assignment object.
user = User.objects.get(email = submission.user)
assignment.apply_personal_deadlines(user)
data = {
"assignment": assignment.to_dict(),
"submission": submission.to_dict(),
"test_harness": test_harness.to_dict()
}
router_send_json(
sheep,
sheep_identity,
FlockMessage("request", data).to_dict()
)
return True
def _rerun_test_harness(assignment):
try:
# Get assignment
assn = Assignment.objects.get(id = ObjectId(assignment))
if not assn.test_harness:
logger.info("Attempting to rerun test harnesses on assignment "
"with no test harnesses")
return
# Form the submissions query
query = {
"assignment": ObjectId(assignment),
"most_recent": True
}
# Grab the most recent submissions from each user.
submissions = list(Submission.objects(**query))
if not submissions:
logger.info("No submissions found for this assignment.")
return
# Send a bunch of test requests to shepherd to be rerun.
for i in submissions:
i.test_request_timestamp = datetime.datetime.now()
i.save()
logger.info("Sent test request to shepherd for %s" % str(i.id))
send_test_request(shepherd_config["PUBLIC_SOCKET"], i.id)
time.sleep(30)
except Exception as e:
logger.error(str(e))
raise
def _rerun_test_harness(assignment):
try:
# Get assignment
assn = Assignment.objects.get(id = ObjectId(assignment))
if not assn.test_harness:
logger.info("Attempting to rerun test harnesses on assignment "
"with no test harnesses")
return
# Form the submissions query
query = {
"assignment": ObjectId(assignment),
"most_recent": True
}
# Grab the most recent submissions from each user.
submissions = list(Submission.objects(**query))
if not submissions:
logger.info("No submissions found for this assignment.")
return
# Send a bunch of test requests to shepherd to be rerun.
for i in submissions:
i.test_request_timestamp = datetime.datetime.now()
i.save()
logger.info("Sent test request to shepherd for %s" % str(i.id))
send_test_request(shepherd_config["PUBLIC_SOCKET"], i.id)
time.sleep(30)
except Exception as e:
logger.error(str(e))
raise
def view_assignment(assignment_id):
simple_archive_form = SimpleArchiveForm()
# Convert the assignment in the URL into an ObjectId
try:
id = ObjectId(assignment_id)
except InvalidId:
logger.info("Invalid Assignment ID requested.")
abort(404)
# Retrieve the assignment
try:
assignment = Assignment.objects.get(id=id)
except Assignment.DoesNotExist:
logger.info("Non-extant ID requested.")
abort(404)
# Get all of the submissions for this assignmnet
submissions = list(Submission.objects(user=current_user.id, assignment=id).order_by("-most_recent", "-timestamp"))
# Add the pretty version of each submissions timestamp
for i in submissions:
i.timestamp_pretty = pretty_time(i.timestamp)
return render_template(
"assignment.html",
now=datetime.datetime.today(),
create_time_element=create_time_element,
assignment=assignment,
submissions=submissions,
simple_archive_form=simple_archive_form,
new_submissions=[v for k, v in get_flashed_messages(with_categories=True) if k == "new_submission"],
)
def _delete_assignments(ids, delete_class):
if delete_class:
delete_class = ObjectId(delete_class)
# Convert all of the IDs we were given to ObjectIDs in one go
ids = [ObjectId(i) for i in ids]
logger.info("Deleting assignments %s.", str(ids))
# Query the database for all the assignments we are supposed to delete
assignments = list(Assignment.objects(id__in = ids))
# Query the database for all the submissions we are supposed to delete, this
# will potentially be an absolutely massive list, so we do not want to
# place all the results into a list immediately like we did for the
# assignments.
submissions = Submission.objects(assignment__in = ids)
# Delete assignments directory on the filesystem
for i in assignments:
try:
shutil.rmtree(
os.path.join(config["SUBMISSION_DIRECTORY"], str(i.id))
)
except OSError as e:
# We don't want to explode if the directory has been deleted for us
# but it is weird so log a warning.
if e.errno == errno.ENOENT:
logger.warning("Assignment directory missing for %s",
str(i.id))
else:
raise
# Actually delete the submissions from the database
Submission.objects(assignment__in = ids).delete()
# Delete the assignments
Assignment.objects(id__in = ids).delete()
if delete_class:
# Unenroll all students in the class
User.objects(classes = delete_class).update(
pull__classes = delete_class
)
# Delete the class
Class.objects(id = delete_class).delete()
def _delete_assignments(ids, delete_class):
if delete_class:
delete_class = ObjectId(delete_class)
# Convert all of the IDs we were given to ObjectIDs in one go
ids = [ObjectId(i) for i in ids]
logger.info("Deleting assignments %s.", str(ids))
# Query the database for all the assignments we are supposed to delete
assignments = list(Assignment.objects(id__in = ids))
# Query the database for all the submissions we are supposed to delete, this
# will potentially be an absolutely massive list, so we do not want to
# place all the results into a list immediately like we did for the
# assignments.
submissions = Submission.objects(assignment__in = ids)
# Delete assignments directory on the filesystem
for i in assignments:
try:
shutil.rmtree(
os.path.join(config["SUBMISSION_DIRECTORY"], str(i.id))
)
except OSError as e:
# We don't want to explode if the directory has been deleted for us
# but it is weird so log a warning.
if e.errno == errno.ENOENT:
logger.warning("Assignment directory missing for %s",
str(i.id))
else:
raise
# Actually delete the submissions from the database
Submission.objects(assignment__in = ids).delete()
# Delete the assignments
Assignment.objects(id__in = ids).delete()
if delete_class:
# Unenroll all students in the class
User.objects(classes = delete_class).update(
pull__classes = delete_class
)
# Delete the class
Class.objects(id = delete_class).delete()
def _delete_assignments(ids, delete_class):
if delete_class:
delete_class = ObjectId(delete_class)
# Convert all of the IDs we were given to ObjectIDs in one go
ids = [ObjectId(i) for i in ids]
logger.debug("Deleting assignments %s.", ids)
# Query the database for all the assignments we are supposed to delete
assignments = list(Assignment.objects(id__in = ids))
# Query the database for all the submissions we are supposed to delete, this
# will potentially be an absolutely massive list, so we do not want to
# place all the results into a list immediately like we did for the
# assignments.
submissions = Submission.objects(assignment__in = ids)
# Go through all of the submissions and delete the files from the
# filesystem. We will tell the database to delete all of the submissions
# in one go afterwards.
for i in submissions:
# Delete the submission on the filesystem.
try:
shutil.rmtree(
os.path.join(config["SUBMISSION_DIRECTORY"], str(i.id))
)
except OSError as e:
logger.warn(
"Failed to delete submission with id %s: %s", str(i.id), str(e)
)
# Actually delete the submissions from the database
Submission.objects(assignment__in = ids).delete()
# Delete the assignments
Assignment.objects(id__in = ids).delete()
if delete_class:
# Unenroll all students in the class
User.objects(classes = delete_class).update(
pull__classes = delete_class
)
# Delete the class
Class.objects(id = delete_class).delete()
def _delete_assignments(ids, delete_class):
if delete_class:
delete_class = ObjectId(delete_class)
# Convert all of the IDs we were given to ObjectIDs in one go
ids = [ObjectId(i) for i in ids]
logger.debug("Deleting assignments %s.", ids)
# Query the database for all the assignments we are supposed to delete
assignments = list(Assignment.objects(id__in=ids))
# Query the database for all the submissions we are supposed to delete, this
# will potentially be an absolutely massive list, so we do not want to
# place all the results into a list immediately like we did for the
# assignments.
submissions = Submission.objects(assignment__in=ids)
# Go through all of the submissions and delete the files from the
# filesystem. We will tell the database to delete all of the submissions
# in one go afterwards.
for i in submissions:
# Delete the submission on the filesystem.
try:
shutil.rmtree(
os.path.join(config["SUBMISSION_DIRECTORY"], str(i.id)))
except OSError as e:
logger.warn("Failed to delete submission with id %s: %s",
str(i.id), str(e))
# Actually delete the submissions from the database
Submission.objects(assignment__in=ids).delete()
# Delete the assignments
Assignment.objects(id__in=ids).delete()
if delete_class:
# Unenroll all students in the class
User.objects(classes=delete_class).update(pull__classes=delete_class)
# Delete the class
Class.objects(id=delete_class).delete()
def view_assignment(assignment_id):
simple_archive_form = SimpleArchiveForm()
# Convert the assignment in the URL into an ObjectId
try:
id = ObjectId(assignment_id)
except InvalidId:
app.logger.debug("Invalid ID (%s)" % str(id))
abort(404)
# Retrieve the assignment
try:
assignment = Assignment.objects.get(id=id)
except Assignment.DoesNotExist:
app.logger.debug("Non-extant ID (%s)" % str(id))
abort(404)
# Get all of the submissions for this assignmnet
submissions = list(Submission.objects(user=current_user.id, assignment=id))
# Add the pretty version of each submissions timestamp
for i in submissions:
i.timestamp_pretty = pretty_time(i.timestamp)
return render_template(
"assignment.html",
now=datetime.datetime.today(),
create_time_element=create_time_element,
assignment=assignment,
submissions=submissions,
simple_archive_form=simple_archive_form,
new_submissions=[
v for k, v in get_flashed_messages(with_categories=True)
if k == "new_submission"
])
def view_assignment(assignment_id):
simple_archive_form = SimpleArchiveForm()
# Convert the assignment in the URL into an ObjectId
try:
id = ObjectId(assignment_id)
except InvalidId:
logger.info("Invalid Assignment ID requested.")
abort(404)
# Retrieve the assignment
try:
assignment = Assignment.objects.get(id=id)
except Assignment.DoesNotExist:
logger.info("Non-extant ID requested.")
abort(404)
assignment.apply_personal_deadlines(current_user)
# Get all of the submissions for this assignment
submissions = list(
Submission.objects(user=current_user.id,
assignment=id).order_by("-most_recent",
"-timestamp"))
# Get student submissions if being viewed as teacher
student_submissions = []
students = []
if current_user.account_type in ["teacher", "teaching_assistant"]:
students = list(
User.objects(classes=assignment.for_class,
account_type="student").order_by("-email"))
student_submissions = list(
Submission.objects(user__in=[i.email for i in students],
assignment=assignment_id,
most_recent=True))
test_results = list(
TestResult.objects(
id__in=[i.test_results for i in submissions if i.test_results]))
# Match test results to submissions
for i in submissions:
for j in test_results:
if i.test_results == j.id:
i.test_results_obj = j
# Current time to be compared to submission test_request_timestamp
now = datetime.datetime.now()
# Flag to set refresh trigger if user is waiting on test results
wait_and_refresh = False
# Add the pretty version of each submissions timestamp
for i in submissions:
i.timestamp_pretty = pretty_time(i.timestamp)
i.status = "Submitted"
# If the user submitted a test request and there aren't any results
if (i.test_request_timestamp and not i.test_results):
timedelta = now - i.test_request_timestamp
i.show_resubmit = (timedelta > config["STUDENT_RETRY_INTERVAL"])
if not i.show_resubmit:
i.status = "Waiting for test results..."
else:
i.status = "Test request timed out"
elif (i.test_results and i.test_results_obj.failed):
i.status = "Tests Failed"
i.show_resubmit = True
elif (i.test_results and not i.test_results_obj.failed):
i.status = "Tests Completed"
wait_and_refresh = \
any(i.status == "Waiting for test results..." for i in submissions)
return render_template(
"assignment.html",
now=datetime.datetime.today(),
create_time_element=create_time_element,
assignment=assignment,
submissions=submissions,
simple_archive_form=simple_archive_form,
wait_and_refresh=wait_and_refresh,
new_submissions=[
v for k, v in get_flashed_messages(with_categories=True)
if k == "new_submission"
],
view_as_teacher=(current_user.account_type
in ["teacher", "teaching_assistant"]),
students=students,
student_submissions=student_submissions,
enumerate=enumerate)
"not accepted."
)
form = SimpleArchiveForm()
if not form.validate_on_submit():
flash("The files you passed in were invalid.", category = "error")
return redirect(redirect_to)
if not [i for i in form.archive.entries if i.data.filename]:
flash("You did not submit any files.", category = "error")
return redirect(redirect_to)
new_submission = Submission(
assignment = id,
user = current_user.id,
timestamp = datetime.datetime.now(),
marked_for_grading = True,
most_recent = True
)
new_submission.id = ObjectId()
# Craft a unique directory path where we will store the new submission. We
# are guarenteed an ObjectId is unique. However we are not guarenteed that
# we will have the proper permissions and that we will be able to make the
# directory thus this could error because of that.
new_submission.testables = os.path.join(
app.config["SUBMISSION_DIRECTORY"], str(new_submission.id)
)
os.makedirs(new_submission.testables)
# Save each file the user uploaded into the submissions directory
def _create_assignment_csv(csv_id, requester, assignment):
csv_id = ObjectId(csv_id)
csv_file = temp_directory = ""
# Find any expired archives and remove them
deleted_files = []
for i in CSV.objects(expires__lt = datetime.datetime.today()):
deleted_files.append(i.file_location)
if i.file_location:
try:
os.remove(i.file_location)
except OSError as e:
logger.warning(
"Could not remove expired csv file at %s: %s.",
i.file_location, str(e)
)
i.delete()
if deleted_files:
logger.info("Deleted csv files %s.", str(deleted_files))
# This is the CSV object that will be added to the database
new_csv = CSV(
id = csv_id,
requester = requester
)
temp_directory = csv_file = None
try:
assn = Assignment.objects.get(id = ObjectId(assignment))
# Grab all student users for this class.
users = list(
User.objects(
account_type = "student",
classes = assn.for_class
)
)
# Form the query
query = {
"assignment": ObjectId(assignment),
"most_recent": True,
"user__in": [i.id for i in users]
}
# Grab the most recent submissions from each user.
submissions = list(Submission.objects(**query))
# Create the actual csv file.
csv_file = open(os.path.join(config["CSV_DIRECTORY"], str(csv_id)), "w")
for i in submissions:
score = "None"
if i.test_results:
test_result = TestResult.objects.get(id = i.test_results)
score = str(test_result.score)
print >> csv_file, "%s,%s,%s" % \
(i.user, score, i.timestamp.strftime("%Y-%m-%d-%H-%M-%S"))
csv_file.close()
new_csv.file_location = os.path.join(config["CSV_DIRECTORY"], str(csv_id))
new_csv.expires = \
datetime.datetime.today() + config["TEACHER_CSV_LIFETIME"]
new_csv.save(force_insert = True)
except Exception as e:
new_csv.file_location = None
os.remove(os.path.join(config["CSV_DIRECTORY"], str(csv_id)))
new_csv.error_string = str(e)
new_csv.save(force_insert = True)
raise
def view_assignment(assignment_id):
simple_archive_form = SimpleArchiveForm()
# Convert the assignment in the URL into an ObjectId
try:
id = ObjectId(assignment_id)
except InvalidId:
logger.info("Invalid Assignment ID requested.")
abort(404)
# Retrieve the assignment
try:
assignment = Assignment.objects.get(id=id)
except Assignment.DoesNotExist:
logger.info("Non-extant ID requested.")
abort(404)
assignment.apply_personal_deadlines(current_user)
# Get all of the submissions for this assignment
submissions = list(Submission.objects(user=current_user.id, assignment=id).order_by("-most_recent", "-timestamp"))
# Get student submissions if being viewed as teacher
student_submissions = []
students = []
if current_user.account_type in ["teacher", "teaching_assistant"]:
students = list(User.objects(classes=assignment.for_class, account_type="student").order_by("-email"))
student_submissions = list(
Submission.objects(user__in=[i.email for i in students], assignment=assignment_id, most_recent=True)
)
test_results = list(TestResult.objects(id__in=[i.test_results for i in submissions if i.test_results]))
# Match test results to submissions
for i in submissions:
for j in test_results:
if i.test_results == j.id:
i.test_results_obj = j
# Current time to be compared to submission test_request_timestamp
now = datetime.datetime.now()
# Flag to set refresh trigger if user is waiting on test results
wait_and_refresh = False
# Add the pretty version of each submissions timestamp
for i in submissions:
i.timestamp_pretty = pretty_time(i.timestamp)
i.status = "Submitted"
# If the user submitted a test request and there aren't any results
if i.test_request_timestamp and not i.test_results:
timedelta = now - i.test_request_timestamp
i.show_resubmit = timedelta > config["STUDENT_RETRY_INTERVAL"]
if not i.show_resubmit:
i.status = "Waiting for test results..."
else:
i.status = "Test request timed out"
elif i.test_results and i.test_results_obj.failed:
i.status = "Tests Failed"
i.show_resubmit = True
elif i.test_results and not i.test_results_obj.failed:
i.status = "Tests Completed"
wait_and_refresh = any(i.status == "Waiting for test results..." for i in submissions)
return render_template(
"assignment.html",
now=datetime.datetime.today(),
create_time_element=create_time_element,
assignment=assignment,
submissions=submissions,
simple_archive_form=simple_archive_form,
wait_and_refresh=wait_and_refresh,
new_submissions=[v for k, v in get_flashed_messages(with_categories=True) if k == "new_submission"],
view_as_teacher=(current_user.account_type in ["teacher", "teaching_assistant"]),
students=students,
student_submissions=student_submissions,
enumerate=enumerate,
)
def upload_submission(assignment_id):
# Figure out which assignment the user asked for.
try:
id = ObjectId(assignment_id)
assignment = Assignment.objects.get(id = id)
except (InvalidId, Assignment.DoesNotExist) as e:
logger.info("Could not retrieve assignment: %s", str(e))
abort(404)
# Figure out where we should redirect the user to once we're done.
redirect_to = request.args.get("next") or request.referrer
if not is_url_on_site(app, redirect_to):
# Default going back to the assignment screen
redirect_to = url_for(
"view_assignment",
assignment_id = assignment_id
)
assignment.apply_personal_deadlines(current_user)
# Check if the assignment's cutoff date has passed
if assignment.due_cutoff and \
assignment.due_cutoff < datetime.datetime.today():
logger.info("Submission rejected, cutoff date has already passed.")
flash(
"The cutoff date has already passed, your submission was not "
"accepted.", category = "error"
)
return redirect(redirect_to)
form = SimpleArchiveForm()
if not form.validate_on_submit():
logger.info(
"Submission rejected due to internal validation problem."
)
flash(
"Submission rejected due to internal validation problem. Try "
"again.", category = "error"
)
return redirect(redirect_to)
if not [i for i in form.archive.entries if i.data.filename]:
logger.info("Submission rejected. User did not submit any files.")
flash("You did not submit any files.", category = "error")
return redirect(redirect_to)
new_submission = Submission(
assignment = id,
user = current_user.id,
timestamp = datetime.datetime.now(),
test_type = "final" if form.marked_as_final.data else "public",
most_recent = True
)
new_submission.id = ObjectId()
logger.info(str(new_submission.to_dict()))
# Craft a unique directory path where we will store the new submission. We
# are guarenteed an ObjectId is unique. However we are not guarenteed that
# we will have the proper permissions and that we will be able to make the
# directory thus this could error because of that.
new_submission.testables = new_submission.getFilePath()
os.makedirs(new_submission.testables)
# Save each file the user uploaded into the submissions directory
for i in form.archive.entries:
if not i.data.filename:
continue
# Figure out where we want to save the user's file
file_path = os.path.join(
new_submission.testables, secure_filename(i.data.filename)
)
# Do the actual saving
i.data.save(file_path)
new_submission.uploaded_filenames.extend(
secure_filename(i.data.filename) for i in form.archive.entries
if i.data.filename
)
logger.info(
"Succesfully uploaded a new submission (id = %s) with files %s.",
str(new_submission.id),
str(new_submission.uploaded_filenames)
)
# The old "most_recent" submission is no longer the most recent.
Submission.objects(
user = current_user.email,
assignment = id,
most_recent = True
).update(
multi = False,
unset__most_recent = 1
)
if assignment.test_harness:
new_submission.test_request_timestamp = datetime.datetime.now()
logger.info("Sent test request to shepherd for %s" % \
str(new_submission.id))
new_submission.save()
# Tell shepherd to start running tests if there is a test_harness.
if assignment.test_harness:
send_test_request(config["PUBLIC_SOCKET"], new_submission.id)
# Communicate to the next page what submission was just added.
flash(str(new_submission.id), category = "new_submission")
flash(
"Successfully uploaded %s %s." %
(
plural_if("file", len(new_submission.uploaded_filenames)),
pretty_list(new_submission.uploaded_filenames)
),
category = "message"
)
# Everything seems to have gone well
return redirect(redirect_to)
return craft_response(
error="The cutoff date has already passed, your submission was "
"not accepted.")
form = SimpleArchiveForm()
if not form.validate_on_submit():
flash("The files you passed in were invalid.", category="error")
return redirect(redirect_to)
if not [i for i in form.archive.entries if i.data.filename]:
flash("You did not submit any files.", category="error")
return redirect(redirect_to)
new_submission = Submission(assignment=id,
user=current_user.id,
timestamp=datetime.datetime.now(),
marked_for_grading=True,
most_recent=True)
new_submission.id = ObjectId()
# Craft a unique directory path where we will store the new submission. We
# are guarenteed an ObjectId is unique. However we are not guarenteed that
# we will have the proper permissions and that we will be able to make the
# directory thus this could error because of that.
new_submission.testables = os.path.join(app.config["SUBMISSION_DIRECTORY"],
str(new_submission.id))
os.makedirs(new_submission.testables)
# Save each file the user uploaded into the submissions directory
for i in form.archive.entries:
if not i.data.filename:
def _tar_bulk_submissions(archive_id, requester, assignment, email = ""):
archive_id = ObjectId(archive_id)
archive_file = temp_directory = ""
# Find any expired archives and remove them
deleted_files = []
for i in Archive.objects(expires__lt = datetime.datetime.today()):
deleted_files.append(i.file_location)
if i.file_location:
try:
os.remove(i.file_location)
except OSError as e:
logger.warning(
"Could not remove expired archive at %s: %s.",
i.file_location, str(e)
)
i.delete()
if deleted_files:
logger.info("Deleted archives %s.", str(deleted_files))
# This is the archive object we will eventually add to the database
new_archive = Archive(
id = archive_id,
requester = requester,
archive_type = "assignment_package"
)
temp_directory = archive_file = None
try:
# Form the query
query = {"assignment": ObjectId(assignment)}
# Only mention email in the query if it's not None or the empty
# string, otherwise mongo will look for submissions that list the
# user as None or the empty string (which should be exactly none of
# the submission in the system).
if email:
query["user"] = email
# Grab all the submissions
submissions = list(Submission.objects(**query))
if not submissions:
logger.info("No submissions found matching query.")
return
# Organize all the submissions by user name, as this will closely
# match the structure of the archive we will build.
submission_map = {}
for i in submissions:
if i.user in submission_map:
submission_map[i.user].append(i)
else:
submission_map[i.user] = [i]
# Create a temporary directory we will create our archive in.
temp_directory = tempfile.mkdtemp()
# Create our directory tree. Instead of making new folders for each
# submission and copying the user's files over however, we will
# create symlinks to save space and time.
for user, user_submissions in submission_map.items():
# Create a directory for the user
os.makedirs(os.path.join(temp_directory, user))
# Create symlinks for all his submissions. Each symlink is
# named after the submission date.
for i in user_submissions:
time_stamp = i.timestamp.strftime("%Y-%m-%d-%H-%M-%S")
symlink_path = \
os.path.join(temp_directory, user, time_stamp)
# In the highly unlikely event that two of the same user's
# submissions have the same exact time stamp, we'll need to
# add a marker to the end of the timestamp.
marker = 0
while os.path.exists(symlink_path +
("-%d" % marker if marker > 0 else "")):
marker += 1
if marker > 0:
symlink_path += "-%d" % marker
original_path = i.getFilePath()
# Detect if the submission's files are still on the filesystem
if os.path.isdir(original_path):
# Create a symlink pointing to the actual submission
# directory with the name we gnerated
os.symlink(original_path, symlink_path)
else:
# Create an empty text file marking the fact that a
# submissions existed but is no longer available.
open(symlink_path, "w").close()
# Create the actual archive file.
# TODO: Create it in galah's /var/ directory
file_descriptor, archive_file = tempfile.mkstemp(suffix = ".tar.gz")
os.close(file_descriptor)
# Run tar and do the actual archiving. Will block until it's finished.
subprocess.check_call(
[
"tar", "--dereference", "--create", "--gzip", "--directory",
temp_directory, "--file", archive_file
] + submission_map.keys()
)
new_archive.file_location = archive_file
new_archive.expires = \
datetime.datetime.today() + config["TEACHER_ARCHIVE_LIFETIME"]
new_archive.save(force_insert = True)
except Exception as e:
# If we created a temporary archive file we need to delete it.
new_archive.file_location = None
if archive_file:
os.remove(archive_file)
new_archive.error_string = str(e)
new_archive.save(force_insert = True)
raise
finally:
if temp_directory:
shutil.rmtree(temp_directory)
def _zip_bulk_submissions(archive_id, requester, assignment, email = ""):
archive_id = ObjectId(archive_id)
archive_file = temp_directory = ""
# Find any expired archives and remove them
deleted_files = []
for i in Archive.objects(expires__lt = datetime.datetime.today()):
deleted_files.append(i.file_location)
if i.file_location:
try:
os.remove(i.file_location)
except OSError as e:
logger.warning(
"Could not remove expired archive at %s: %s.",
i.file_location, str(e)
)
i.delete()
if deleted_files:
logger.info("Deleted archives %s.", str(deleted_files))
# This is the archive object we will eventually add to the database
new_archive = Archive(
id = archive_id,
requester = requester,
archive_type = "assignment_package"
)
temp_directory = archive_file = None
try:
# Form the query
query = {"assignment": ObjectId(assignment)}
# Only mention email in the query if it's not None or the empty
# string, otherwise mongo will look for submissions that list the
# user as None or the empty string (which should be exactly none of
# the submission in the system).
if email:
query["user"] = email
else:
# Otherwise, we need to be careful not to get teacher/TA submissions.
assn = Assignment.objects.get(id = ObjectId(assignment))
students = User.objects(
account_type="student",
classes = assn.for_class
)
query["user__in"] = [i.id for i in students]
# Grab all the submissions
submissions = list(Submission.objects(**query))
if not submissions:
logger.info("No submissions found matching query.")
return
# Organize all the submissions by user name, as this will closely
# match the structure of the archive we will build.
submission_map = {}
for i in submissions:
if i.user in submission_map:
submission_map[i.user].append(i)
else:
submission_map[i.user] = [i]
# Create a temporary directory we will create our archive in.
temp_directory = tempfile.mkdtemp()
# Create our directory tree. Instead of making new folders for each
# submission and copying the user's files over however, we will
# create symlinks to save space and time.
for user, user_submissions in submission_map.items():
# Create a directory for the user
os.makedirs(os.path.join(temp_directory, user))
# Create symlinks for all his submissions. Each symlink is
# named after the submission date.
for i in user_submissions:
time_stamp = i.timestamp.strftime("%Y-%m-%d-%H-%M-%S")
symlink_path = \
os.path.join(temp_directory, user, time_stamp)
# In the highly unlikely event that two of the same user's
# submissions have the same exact time stamp, we'll need to
# add a marker to the end of the timestamp.
marker = 0
while os.path.exists(symlink_path +
("-%d" % marker if marker > 0 else "")):
marker += 1
if marker > 0:
symlink_path += "-%d" % marker
original_path = i.getFilePath()
# Detect if the submission's files are still on the filesystem
if os.path.isdir(original_path):
# Create a symlink pointing to the actual submission
# directory with the name we gnerated
os.symlink(original_path, symlink_path)
else:
# Create an empty text file marking the fact that a
# submissions existed but is no longer available.
open(symlink_path, "w").close()
# Create the actual archive file.
# TODO: Create it in galah's /var/ directory
file_descriptor, archive_file = tempfile.mkstemp(suffix = ".zip")
os.close(file_descriptor)
# Run zip and do the actual archiving. Will block until it's finished.
zipdir(temp_directory, archive_file)
new_archive.file_location = archive_file
new_archive.expires = \
datetime.datetime.today() + config["TEACHER_ARCHIVE_LIFETIME"]
new_archive.save(force_insert = True)
except Exception as e:
# If we created a temporary archive file we need to delete it.
new_archive.file_location = None
if archive_file:
os.remove(archive_file)
new_archive.error_string = str(e)
new_archive.save(force_insert = True)
raise
finally:
if temp_directory:
shutil.rmtree(temp_directory)
def browse_assignments():
# Grab all the current user's classes
classes = Class.objects(id__in = current_user.classes).only("name")
# Get the current time so we don't have to do it over and over again.
now = datetime.datetime.today()
if "show_all" in request.args:
assignments = list(Assignment.objects(
Q(for_class__in = current_user.classes) &
(Q(hide_until = None) | Q(hide_until__lt = now))
).only("name", "due", "due_cutoff", "for_class"))
else:
assignments = list(Assignment.objects(
Q(for_class__in = current_user.classes) &
(Q(due__gt = now - datetime.timedelta(weeks = 1)) |
Q(due_cutoff__gt = now - datetime.timedelta(weeks = 1))) &
(Q(hide_until = None) | Q(hide_until__lt = now))
).only("name", "due", "due_cutoff", "for_class"))
assignments = [i for i in assignments if
not i.hide_until or i.hide_until < now]
# Get the number of assignments that we could have gotten if we didn't
# limit based on due date.
all_assignments_count = Assignment.objects(
Q(for_class__in = current_user.classes) &
(Q(hide_until = None) | Q(hide_until__lt = now))
).count()
submissions = list(Submission.objects(
user = current_user.email,
assignment__in = [i.id for i in assignments],
most_recent = True
))
# Add a property to all the assignments so the template can display their
# respective class easier. Additionally, add a plain text version of the
# due date
for i in assignments:
try:
i.class_name = next((j.name for j in classes if j.id == i.for_class))
except StopIteration:
logger.error(
"Assignment with id %s references non-existant class with id "
"%s." % (str(i.id, i.for_class))
)
i.class_name = "DNE"
# Figure out the status messages that we want to display to the user.
submitted = next((j for j in submissions if j.assignment == i.id), None)
i.status = i.status_color = None
if submitted:
i.status = (
"You made a submission " +
create_time_element(submitted.timestamp, now)
)
i.status_color = "#84B354"
elif now < i.due:
i.status = "You have not submitted yet"
i.status_color = "#877150"
elif now > i.due and i.due_cutoff and now > i.due_cutoff:
i.status = "You have not submitted yet, and it is too late to do so"
i.status_color = "#E9A400"
elif now > i.due:
i.status = "You have not submitted yet!"
i.status_color = "#FB4313"
# Sort the assignments by due_cutoff or due date if due_cutoff is not
# assigned.
assignments.sort(
key = lambda i: i.due_cutoff if i.due_cutoff else i.due,
reverse = True
)
return render_template(
"assignments.html",
assignments = assignments,
hidden_assignments = -1 if "show_all" in request.args
else all_assignments_count - len(assignments),
create_time_element = create_time_element
)
def browse_assignments():
# Grab all the current user's classes
classes = Class.objects(id__in=current_user.classes).only("name")
# Get the current time so we don't have to do it over and over again.
now = datetime.datetime.today()
if "show_all" in request.args:
assignments = list(
Assignment.objects(
Q(for_class__in=current_user.classes)
& (Q(hide_until=None) | Q(hide_until__lt=now))).only(
"name", "due", "due_cutoff", "for_class"))
else:
assignments = list(
Assignment.objects(
Q(for_class__in=current_user.classes)
& (Q(due__gt=now - datetime.timedelta(weeks=1))
| Q(due_cutoff__gt=now - datetime.timedelta(weeks=1)))
& (Q(hide_until=None) | Q(hide_until__lt=now))).only(
"name", "due", "due_cutoff", "for_class"))
assignments = [
i for i in assignments if not i.hide_until or i.hide_until < now
]
# Get the number of assignments that we could have gotten if we didn't
# limit based on due date.
all_assignments_count = Assignment.objects(
Q(for_class__in=current_user.classes)
& (Q(hide_until=None) | Q(hide_until__lt=now))).count()
submissions = list(
Submission.objects(assignment__in=[i.id for i in assignments],
most_recent=True))
# Add a property to all the assignments so the template can display their
# respective class easier. Additionally, add a plain text version of the
# due date
for i in assignments:
try:
i.class_name = next(
(j.name for j in classes if j.id == i.for_class))
except StopIteration:
app.logger.error("Assignment with id %s references non-existant "
"class with id %s." % (str(i.id, i.for_class)))
i.class_name = "DNE"
# Figure out the status messages that we want to display to the user.
submitted = next((j for j in submissions if j.assignment == i.id),
None)
i.status = i.status_color = None
if submitted:
i.status = ("You made a submission " +
create_time_element(submitted.timestamp, now))
i.status_color = "#84B354"
elif now < i.due:
i.status = "You have not submitted yet"
i.status_color = "#877150"
elif now > i.due and i.due_cutoff and now > i.due_cutoff:
i.status = "You have not submitted yet, and it is too late to do so"
i.status_color = "#E9A400"
elif now > i.due:
i.status = "You have not submitted yet!"
i.status_color = "#FB4313"
# Sort the assignments by due_cutoff or due date if due_cutoff is not
# assigned.
assignments.sort(key=lambda i: i.due_cutoff if i.due_cutoff else i.due,
reverse=True)
return render_template("assignments.html",
assignments=assignments,
hidden_assignments=-1 if "show_all" in request.args
else all_assignments_count - len(assignments),
create_time_element=create_time_element)
def _tar_bulk_submissions(archive_id, requester, assignment, email = ""):
archive_id = ObjectId(archive_id)
archive_file = temp_directory = ""
# Find any expired archives and remove them
for i in Archive.objects(expires__lt = datetime.datetime.today()):
if i.file_location:
logger.debug("Erasing old archive at '%s'." % i.file_location)
try:
os.remove(i.file_location)
except OSError:
logger.warn(
"Could not remove expired archive at %s.",
i.file_location
)
i.delete()
# This is the archive object we will eventually add to the database
new_archive = Archive(
id = archive_id,
requester = requester,
archive_type = "assignment_package"
)
temp_directory = archive_file = None
try:
# Form the query
query = {"assignment": ObjectId(assignment)}
# Only mention email in the query if it's not None or the empty
# string, otherwise mongo will look for submissions that list the
# user as None or the empty string (which should be exactly none of
# the submission in the system).
if email:
query["user"] = email
# Grab all the submissions
submissions = list(Submission.objects(**query))
if not submissions:
raise RuntimeError("No submissions found matching query.")
# Organize all the submissions by user name, as this will closely
# match the structure of the archive we will build.
submission_map = {}
for i in submissions:
if i.user in submission_map:
submission_map[i.user].append(i)
else:
submission_map[i.user] = [i]
# Create a temporary directory we will create our archive in.
temp_directory = tempfile.mkdtemp()
# Create our directory tree. Instead of making new folders for each
# submission and copying the user's files over however, we will
# create symlinks to save space and time.
for user, user_submissions in submission_map.items():
# Create a directory for the user
os.makedirs(os.path.join(temp_directory, user))
# Create symlinks for all his submissions. Each symlink is
# named after the submission date.
for i in user_submissions:
time_stamp = i.timestamp.strftime("%Y-%m-%d-%H-%M-%S")
symlink_path = \
os.path.join(temp_directory, user, time_stamp)
# In the highly unlikely event that two of the same user's
# submissions have the same exact time stamp, we'll need to
# add a marker to the end of the timestamp.
marker = 0
while os.path.isfile(symlink_path +
("-%d" % marker if marker > 0 else "")):
marker += 1
if marker > 0:
symlink_path += "-%d" % marker
original_path = \
os.path.join(config["SUBMISSION_DIRECTORY"], str(i.id))
# Detect if the submission's files are still on the filesystem
if os.path.isdir(original_path):
# Create a symlink pointing to the actual submission
# directory with the name we gnerated
os.symlink(original_path, symlink_path)
else:
# Create an empty text file marking the fact that a
# submissions existed but is no longer available.
open(symlink_path, "w").close()
# Create the actual archive file.
# TODO: Create it in galah's /var/ directory
file_descriptor, archive_file = tempfile.mkstemp(suffix = ".tar.gz")
os.close(file_descriptor)
# Run tar and do the actual archiving. Will block until it's finished.
subprocess.check_call(
[
"tar", "--dereference", "--create", "--gzip", "--directory",
temp_directory, "--file", archive_file
] + submission_map.keys()
)
new_archive.file_location = archive_file
new_archive.expires = \
datetime.datetime.today() + datetime.timedelta(hours = 2)
new_archive.save(force_insert = True)
except Exception as e:
logger.exception("An error occured while creating an archive.")
# If we created a temporary archive file we need to delete it.
new_archive.file_location = None
if archive_file:
os.remove(archive_file)
new_archive.error_string = str(e)
new_archive.save(force_insert = True)
finally:
if temp_directory:
shutil.rmtree(temp_directory)
def upload_submission(assignment_id):
# Figure out which assignment the user asked for.
try:
id = ObjectId(assignment_id)
assignment = Assignment.objects.get(id=id)
except (InvalidId, Assignment.DoesNotExist) as e:
logger.info("Could not retrieve assignment: %s", str(e))
abort(404)
# Figure out where we should redirect the user to once we're done.
redirect_to = request.args.get("next") or request.referrer
if not is_url_on_site(app, redirect_to):
# Default going back to the assignment screen
redirect_to = url_for("view_assignment", assignment_id=assignment_id)
assignment.apply_personal_deadlines(current_user)
# Check if the assignment's cutoff date has passed
if assignment.due_cutoff and \
assignment.due_cutoff < datetime.datetime.today():
logger.info("Submission rejected, cutoff date has already passed.")
flash(
"The cutoff date has already passed, your submission was not "
"accepted.",
category="error")
return redirect(redirect_to)
form = SimpleArchiveForm()
if not form.validate_on_submit():
logger.info("Submission rejected due to internal validation problem.")
flash(
"Submission rejected due to internal validation problem. Try "
"again.",
category="error")
return redirect(redirect_to)
if not [i for i in form.archive.entries if i.data.filename]:
logger.info("Submission rejected. User did not submit any files.")
flash("You did not submit any files.", category="error")
return redirect(redirect_to)
new_submission = Submission(
assignment=id,
user=current_user.id,
timestamp=datetime.datetime.now(),
test_type="final" if form.marked_as_final.data else "public",
most_recent=True)
new_submission.id = ObjectId()
logger.info(str(new_submission.to_dict()))
# Craft a unique directory path where we will store the new submission. We
# are guarenteed an ObjectId is unique. However we are not guarenteed that
# we will have the proper permissions and that we will be able to make the
# directory thus this could error because of that.
new_submission.testables = new_submission.getFilePath()
os.makedirs(new_submission.testables)
# Save each file the user uploaded into the submissions directory
for i in form.archive.entries:
if not i.data.filename:
continue
# Figure out where we want to save the user's file
file_path = os.path.join(new_submission.testables,
secure_filename(i.data.filename))
# Do the actual saving
i.data.save(file_path)
new_submission.uploaded_filenames.extend(
secure_filename(i.data.filename) for i in form.archive.entries
if i.data.filename)
logger.info(
"Succesfully uploaded a new submission (id = %s) with files %s.",
str(new_submission.id), str(new_submission.uploaded_filenames))
# The old "most_recent" submission is no longer the most recent.
Submission.objects(user=current_user.email,
assignment=id,
most_recent=True).update(multi=False,
unset__most_recent=1)
if assignment.test_harness:
new_submission.test_request_timestamp = datetime.datetime.now()
logger.info("Sent test request to shepherd for %s" % \
str(new_submission.id))
new_submission.save()
# Tell shepherd to start running tests if there is a test_harness.
if assignment.test_harness:
send_test_request(config["PUBLIC_SOCKET"], new_submission.id)
# Communicate to the next page what submission was just added.
flash(str(new_submission.id), category="new_submission")
flash("Successfully uploaded %s %s." %
(plural_if("file", len(new_submission.uploaded_filenames)),
pretty_list(new_submission.uploaded_filenames)),
category="message")
# Everything seems to have gone well
return redirect(redirect_to)
def _create_assignment_csv(csv_id, requester, assignment):
csv_id = ObjectId(csv_id)
csv_file = temp_directory = ""
# Find any expired archives and remove them
deleted_files = []
for i in CSV.objects(expires__lt=datetime.datetime.today()):
deleted_files.append(i.file_location)
if i.file_location:
try:
os.remove(i.file_location)
except OSError as e:
logger.warning("Could not remove expired csv file at %s: %s.",
i.file_location, str(e))
i.delete()
if deleted_files:
logger.info("Deleted csv files %s.", str(deleted_files))
# This is the CSV object that will be added to the database
new_csv = CSV(id=csv_id, requester=requester)
temp_directory = csv_file = None
try:
assn = Assignment.objects.get(id=ObjectId(assignment))
# Grab all student users for this class.
users = list(
User.objects(account_type="student", classes=assn.for_class))
# Form the query
query = {
"assignment": ObjectId(assignment),
"most_recent": True,
"user__in": [i.id for i in users]
}
# Grab the most recent submissions from each user.
submissions = list(Submission.objects(**query))
# Create the actual csv file.
csv_file = open(os.path.join(config["CSV_DIRECTORY"], str(csv_id)),
"w")
for i in submissions:
score = "None"
if i.test_results:
test_result = TestResult.objects.get(id=i.test_results)
score = str(test_result.score)
print >> csv_file, "%s,%s,%s" % \
(i.user, score, i.timestamp.strftime("%Y-%m-%d-%H-%M-%S"))
csv_file.close()
new_csv.file_location = os.path.join(config["CSV_DIRECTORY"],
str(csv_id))
new_csv.expires = \
datetime.datetime.today() + config["TEACHER_CSV_LIFETIME"]
new_csv.save(force_insert=True)
except Exception as e:
new_csv.file_location = None
os.remove(os.path.join(config["CSV_DIRECTORY"], str(csv_id)))
new_csv.error_string = str(e)
new_csv.save(force_insert=True)
raise
def _create_gradebook_csv(csv_id, requester, class_id, fill=0):
csv_id = ObjectId(csv_id)
csv_file = temp_directory = ""
# Find any expired archives and remove them
deleted_files = []
for i in CSV.objects(expires__lt = datetime.datetime.today()):
deleted_files.append(i.file_location)
if i.file_location:
try:
os.remove(i.file_location)
except OSError as e:
logger.warning(
"Could not remove expired csv file at %s: %s.",
i.file_location, str(e)
)
i.delete()
if deleted_files:
logger.info("Deleted csv files %s.", str(deleted_files))
# This is the CSV object that will be added to the database
new_csv = CSV(
id = csv_id,
requester = requester
)
temp_directory = csv_file = None
try:
# Create the actual csv file.
csv_file = open(os.path.join(config["CSV_DIRECTORY"], str(csv_id)), "w")
the_class = Class.objects.get(id = ObjectId(class_id))
# Grab all assignments in this class
assns = list(
Assignment.objects(for_class = the_class.id)
)
print >> csv_file, "%s,%s" % \
("Username", ",".join('"{0}"'.format(i.name) for i in assns))
# Grab all student users for this class.
users = list(
User.objects(
account_type = "student",
classes = the_class.id
)
)
assn_ids = [i.id for i in assns]
for user in users:
# Query for user's most recent submissions in the known assignments
query = {
"assignment__in": assn_ids,
"most_recent": True,
"user": user.id
}
submissions = list(Submission.objects(**query))
# Initialize each assignment score to empty at first.
assn_to_score = OrderedDict((i, str(fill)) for i in assn_ids)
# Go through submissions, associating scores with assignment
for sub in submissions:
if sub.test_results:
test_result = TestResult.objects.get(id = sub.test_results)
if test_result.score is not None:
assn_to_score[sub.assignment] = str(test_result.score)
# Write gradebook results to csv file.
print >> csv_file, "%s,%s" % \
(user.email, ",".join(assn_to_score.values()))
csv_file.close()
new_csv.file_location = os.path.join(config["CSV_DIRECTORY"], str(csv_id))
new_csv.expires = \
datetime.datetime.today() + config["TEACHER_CSV_LIFETIME"]
new_csv.save(force_insert = True)
except Exception as e:
new_csv.file_location = None
os.remove(os.path.join(config["CSV_DIRECTORY"], str(csv_id)))
new_csv.error_string = str(e)
new_csv.save(force_insert = True)
raise