Exemple #1
0
def match_found(flock_manager, sheep_identity, request):
    logger.info(
        "Sending test request for submission [%s] to sheep [%s].",
        request.submission_id,
        repr(sheep_identity)
    )

    # Get submission and test harness to send to sheep
    submission = Submission.objects(id = request.submission_id).exclude(
        "most_recent",
        "uploaded_filenames"
    ).first()
    assignment = Assignment.objects.get(id = submission.assignment)
    test_harness = TestHarness.objects.get(id = assignment.test_harness)

    # Apply any personal deadlines to the assignment object.
    user = User.objects.get(email = submission.user)
    assignment.apply_personal_deadlines(user)

    data = {
        "assignment": assignment.to_dict(),
        "submission": submission.to_dict(),
        "test_harness": test_harness.to_dict()
    }

    router_send_json(
        sheep,
        sheep_identity,
        FlockMessage("request", data).to_dict()
    )

    return True
Exemple #2
0
def _rerun_test_harness(assignment):
    try:
        # Get assignment
        assn = Assignment.objects.get(id = ObjectId(assignment))

        if not assn.test_harness:
            logger.info("Attempting to rerun test harnesses on assignment "
                        "with no test harnesses")
            return

        # Form the submissions query
        query = {
            "assignment": ObjectId(assignment),
            "most_recent": True
        }

        # Grab the most recent submissions from each user.
        submissions = list(Submission.objects(**query))

        if not submissions:
            logger.info("No submissions found for this assignment.")
            return

        # Send a bunch of test requests to shepherd to be rerun.
        for i in submissions:
            i.test_request_timestamp = datetime.datetime.now()
            i.save()
            logger.info("Sent test request to shepherd for %s" % str(i.id))
            send_test_request(shepherd_config["PUBLIC_SOCKET"], i.id)

            time.sleep(30)
    except Exception as e:
        logger.error(str(e))

        raise
Exemple #3
0
def _rerun_test_harness(assignment):
    try:
        # Get assignment
        assn = Assignment.objects.get(id = ObjectId(assignment))

        if not assn.test_harness:
            logger.info("Attempting to rerun test harnesses on assignment "
                        "with no test harnesses")
            return

        # Form the submissions query
        query = {
            "assignment": ObjectId(assignment),
            "most_recent": True
        }

        # Grab the most recent submissions from each user.
        submissions = list(Submission.objects(**query))

        if not submissions:
            logger.info("No submissions found for this assignment.")
            return

        # Send a bunch of test requests to shepherd to be rerun.
        for i in submissions:
            i.test_request_timestamp = datetime.datetime.now()
            i.save()
            logger.info("Sent test request to shepherd for %s" % str(i.id))
            send_test_request(shepherd_config["PUBLIC_SOCKET"], i.id)

            time.sleep(30)
    except Exception as e:
        logger.error(str(e))

        raise
Exemple #4
0
def view_assignment(assignment_id):
    simple_archive_form = SimpleArchiveForm()

    # Convert the assignment in the URL into an ObjectId
    try:
        id = ObjectId(assignment_id)
    except InvalidId:
        logger.info("Invalid Assignment ID requested.")

        abort(404)

    # Retrieve the assignment
    try:
        assignment = Assignment.objects.get(id=id)
    except Assignment.DoesNotExist:
        logger.info("Non-extant ID requested.")

        abort(404)

    # Get all of the submissions for this assignmnet
    submissions = list(Submission.objects(user=current_user.id, assignment=id).order_by("-most_recent", "-timestamp"))

    # Add the pretty version of each submissions timestamp
    for i in submissions:
        i.timestamp_pretty = pretty_time(i.timestamp)

    return render_template(
        "assignment.html",
        now=datetime.datetime.today(),
        create_time_element=create_time_element,
        assignment=assignment,
        submissions=submissions,
        simple_archive_form=simple_archive_form,
        new_submissions=[v for k, v in get_flashed_messages(with_categories=True) if k == "new_submission"],
    )
Exemple #5
0
def _delete_assignments(ids, delete_class):
    if delete_class:
        delete_class = ObjectId(delete_class)

    # Convert all of the IDs we were given to ObjectIDs in one go
    ids = [ObjectId(i) for i in ids]

    logger.info("Deleting assignments %s.", str(ids))

    # Query the database for all the assignments we are supposed to delete
    assignments = list(Assignment.objects(id__in = ids))

    # Query the database for all the submissions we are supposed to delete, this
    # will potentially be an absolutely massive list, so we do not want to
    # place all the results into a list immediately like we did for the
    # assignments.
    submissions = Submission.objects(assignment__in = ids)

    # Delete assignments directory on the filesystem
    for i in assignments:
        try:
            shutil.rmtree(
                os.path.join(config["SUBMISSION_DIRECTORY"], str(i.id))
            )
        except OSError as e:
            # We don't want to explode if the directory has been deleted for us
            # but it is weird so log a warning.
            if e.errno == errno.ENOENT:
                logger.warning("Assignment directory missing for %s",
                    str(i.id))
            else:
                raise

    # Actually delete the submissions from the database
    Submission.objects(assignment__in = ids).delete()

    # Delete the assignments
    Assignment.objects(id__in = ids).delete()

    if delete_class:
        # Unenroll all students in the class
        User.objects(classes = delete_class).update(
            pull__classes = delete_class
        )

        # Delete the class
        Class.objects(id = delete_class).delete()
Exemple #6
0
def _delete_assignments(ids, delete_class):
    if delete_class:
        delete_class = ObjectId(delete_class)

    # Convert all of the IDs we were given to ObjectIDs in one go
    ids = [ObjectId(i) for i in ids]

    logger.info("Deleting assignments %s.", str(ids))

    # Query the database for all the assignments we are supposed to delete
    assignments = list(Assignment.objects(id__in = ids))

    # Query the database for all the submissions we are supposed to delete, this
    # will potentially be an absolutely massive list, so we do not want to
    # place all the results into a list immediately like we did for the
    # assignments.
    submissions = Submission.objects(assignment__in = ids)

    # Delete assignments directory on the filesystem
    for i in assignments:
        try:
            shutil.rmtree(
                os.path.join(config["SUBMISSION_DIRECTORY"], str(i.id))
            )
        except OSError as e:
            # We don't want to explode if the directory has been deleted for us
            # but it is weird so log a warning.
            if e.errno == errno.ENOENT:
                logger.warning("Assignment directory missing for %s",
                    str(i.id))
            else:
                raise

    # Actually delete the submissions from the database
    Submission.objects(assignment__in = ids).delete()

    # Delete the assignments
    Assignment.objects(id__in = ids).delete()

    if delete_class:
        # Unenroll all students in the class
        User.objects(classes = delete_class).update(
            pull__classes = delete_class
        )

        # Delete the class
        Class.objects(id = delete_class).delete()
Exemple #7
0
def _delete_assignments(ids, delete_class):
    if delete_class:
        delete_class = ObjectId(delete_class)

    # Convert all of the IDs we were given to ObjectIDs in one go
    ids = [ObjectId(i) for i in ids]

    logger.debug("Deleting assignments %s.", ids)

    # Query the database for all the assignments we are supposed to delete
    assignments = list(Assignment.objects(id__in = ids))

    # Query the database for all the submissions we are supposed to delete, this
    # will potentially be an absolutely massive list, so we do not want to
    # place all the results into a list immediately like we did for the
    # assignments.
    submissions = Submission.objects(assignment__in = ids)

    # Go through all of the submissions and delete the files from the
    # filesystem. We will tell the database to delete all of the submissions
    # in one go afterwards.
    for i in submissions:
        # Delete the submission on the filesystem.
        try:
            shutil.rmtree(
                os.path.join(config["SUBMISSION_DIRECTORY"], str(i.id))
            )
        except OSError as e:
            logger.warn(
                "Failed to delete submission with id %s: %s", str(i.id), str(e)
            )

    # Actually delete the submissions from the database
    Submission.objects(assignment__in = ids).delete()

    # Delete the assignments
    Assignment.objects(id__in = ids).delete()

    if delete_class:
        # Unenroll all students in the class
        User.objects(classes = delete_class).update(
            pull__classes = delete_class
        )

        # Delete the class
        Class.objects(id = delete_class).delete()
Exemple #8
0
def _delete_assignments(ids, delete_class):
    if delete_class:
        delete_class = ObjectId(delete_class)

    # Convert all of the IDs we were given to ObjectIDs in one go
    ids = [ObjectId(i) for i in ids]

    logger.debug("Deleting assignments %s.", ids)

    # Query the database for all the assignments we are supposed to delete
    assignments = list(Assignment.objects(id__in=ids))

    # Query the database for all the submissions we are supposed to delete, this
    # will potentially be an absolutely massive list, so we do not want to
    # place all the results into a list immediately like we did for the
    # assignments.
    submissions = Submission.objects(assignment__in=ids)

    # Go through all of the submissions and delete the files from the
    # filesystem. We will tell the database to delete all of the submissions
    # in one go afterwards.
    for i in submissions:
        # Delete the submission on the filesystem.
        try:
            shutil.rmtree(
                os.path.join(config["SUBMISSION_DIRECTORY"], str(i.id)))
        except OSError as e:
            logger.warn("Failed to delete submission with id %s: %s",
                        str(i.id), str(e))

    # Actually delete the submissions from the database
    Submission.objects(assignment__in=ids).delete()

    # Delete the assignments
    Assignment.objects(id__in=ids).delete()

    if delete_class:
        # Unenroll all students in the class
        User.objects(classes=delete_class).update(pull__classes=delete_class)

        # Delete the class
        Class.objects(id=delete_class).delete()
Exemple #9
0
def view_assignment(assignment_id):
    simple_archive_form = SimpleArchiveForm()

    # Convert the assignment in the URL into an ObjectId
    try:
        id = ObjectId(assignment_id)
    except InvalidId:
        app.logger.debug("Invalid ID (%s)" % str(id))

        abort(404)

    # Retrieve the assignment
    try:
        assignment = Assignment.objects.get(id=id)
    except Assignment.DoesNotExist:
        app.logger.debug("Non-extant ID (%s)" % str(id))

        abort(404)

    # Get all of the submissions for this assignmnet
    submissions = list(Submission.objects(user=current_user.id, assignment=id))

    # Add the pretty version of each submissions timestamp
    for i in submissions:
        i.timestamp_pretty = pretty_time(i.timestamp)

    return render_template(
        "assignment.html",
        now=datetime.datetime.today(),
        create_time_element=create_time_element,
        assignment=assignment,
        submissions=submissions,
        simple_archive_form=simple_archive_form,
        new_submissions=[
            v for k, v in get_flashed_messages(with_categories=True)
            if k == "new_submission"
        ])
Exemple #10
0
def view_assignment(assignment_id):
    simple_archive_form = SimpleArchiveForm()

    # Convert the assignment in the URL into an ObjectId
    try:
        id = ObjectId(assignment_id)
    except InvalidId:
        logger.info("Invalid Assignment ID requested.")

        abort(404)

    # Retrieve the assignment
    try:
        assignment = Assignment.objects.get(id=id)
    except Assignment.DoesNotExist:
        logger.info("Non-extant ID requested.")

        abort(404)

    assignment.apply_personal_deadlines(current_user)

    # Get all of the submissions for this assignment
    submissions = list(
        Submission.objects(user=current_user.id,
                           assignment=id).order_by("-most_recent",
                                                   "-timestamp"))

    # Get student submissions if being viewed as teacher
    student_submissions = []
    students = []
    if current_user.account_type in ["teacher", "teaching_assistant"]:
        students = list(
            User.objects(classes=assignment.for_class,
                         account_type="student").order_by("-email"))

        student_submissions = list(
            Submission.objects(user__in=[i.email for i in students],
                               assignment=assignment_id,
                               most_recent=True))

    test_results = list(
        TestResult.objects(
            id__in=[i.test_results for i in submissions if i.test_results]))

    # Match test results to submissions
    for i in submissions:
        for j in test_results:
            if i.test_results == j.id:
                i.test_results_obj = j

    # Current time to be compared to submission test_request_timestamp
    now = datetime.datetime.now()

    # Flag to set refresh trigger if user is waiting on test results
    wait_and_refresh = False

    # Add the pretty version of each submissions timestamp
    for i in submissions:
        i.timestamp_pretty = pretty_time(i.timestamp)
        i.status = "Submitted"

        # If the user submitted a test request and there aren't any results
        if (i.test_request_timestamp and not i.test_results):
            timedelta = now - i.test_request_timestamp
            i.show_resubmit = (timedelta > config["STUDENT_RETRY_INTERVAL"])
            if not i.show_resubmit:
                i.status = "Waiting for test results..."
            else:
                i.status = "Test request timed out"
        elif (i.test_results and i.test_results_obj.failed):
            i.status = "Tests Failed"
            i.show_resubmit = True
        elif (i.test_results and not i.test_results_obj.failed):
            i.status = "Tests Completed"

    wait_and_refresh = \
        any(i.status == "Waiting for test results..." for i in submissions)

    return render_template(
        "assignment.html",
        now=datetime.datetime.today(),
        create_time_element=create_time_element,
        assignment=assignment,
        submissions=submissions,
        simple_archive_form=simple_archive_form,
        wait_and_refresh=wait_and_refresh,
        new_submissions=[
            v for k, v in get_flashed_messages(with_categories=True)
            if k == "new_submission"
        ],
        view_as_teacher=(current_user.account_type
                         in ["teacher", "teaching_assistant"]),
        students=students,
        student_submissions=student_submissions,
        enumerate=enumerate)
Exemple #11
0
                    "not accepted."
        )

    form = SimpleArchiveForm()
    if not form.validate_on_submit():
        flash("The files you passed in were invalid.", category = "error")
        return redirect(redirect_to)

    if not [i for i in form.archive.entries if i.data.filename]:
        flash("You did not submit any files.", category = "error")
        return redirect(redirect_to)

    new_submission = Submission(
        assignment = id,
        user = current_user.id,
        timestamp = datetime.datetime.now(),
        marked_for_grading = True,
        most_recent = True
    )
    new_submission.id = ObjectId()

    # Craft a unique directory path where we will store the new submission. We
    # are guarenteed an ObjectId is unique. However we are not guarenteed that
    # we will have the proper permissions and that we will be able to make the
    # directory thus this could error because of that.
    new_submission.testables = os.path.join(
        app.config["SUBMISSION_DIRECTORY"], str(new_submission.id)
    )
    os.makedirs(new_submission.testables)

    # Save each file the user uploaded into the submissions directory
def _create_assignment_csv(csv_id, requester, assignment):
    csv_id = ObjectId(csv_id)

    csv_file = temp_directory = ""

    # Find any expired archives and remove them
    deleted_files = []
    for i in CSV.objects(expires__lt = datetime.datetime.today()):
        deleted_files.append(i.file_location)

        if i.file_location:
            try:
                os.remove(i.file_location)
            except OSError as e:
                logger.warning(
                    "Could not remove expired csv file at %s: %s.",
                    i.file_location, str(e)
                )

        i.delete()

    if deleted_files:
        logger.info("Deleted csv files %s.", str(deleted_files))

    # This is the CSV object that will be added to the database
    new_csv = CSV(
        id = csv_id,
        requester = requester
    )

    temp_directory = csv_file = None
    try:
        assn = Assignment.objects.get(id = ObjectId(assignment))

        # Grab all student users for this class.
        users = list(
            User.objects(
                account_type = "student",
                classes = assn.for_class
            )
        )

        # Form the query
        query = {
            "assignment": ObjectId(assignment),
            "most_recent": True,
            "user__in": [i.id for i in users]
        }

        # Grab the most recent submissions from each user.
        submissions = list(Submission.objects(**query))

        # Create the actual csv file.
        csv_file = open(os.path.join(config["CSV_DIRECTORY"], str(csv_id)), "w")

        for i in submissions:
            score = "None"
            if i.test_results:
                test_result = TestResult.objects.get(id = i.test_results)
                score = str(test_result.score)

            print >> csv_file, "%s,%s,%s" % \
                (i.user, score, i.timestamp.strftime("%Y-%m-%d-%H-%M-%S"))

        csv_file.close()

        new_csv.file_location = os.path.join(config["CSV_DIRECTORY"], str(csv_id))

        new_csv.expires = \
            datetime.datetime.today() + config["TEACHER_CSV_LIFETIME"]

        new_csv.save(force_insert = True)
    except Exception as e:
        new_csv.file_location = None
        os.remove(os.path.join(config["CSV_DIRECTORY"], str(csv_id)))

        new_csv.error_string = str(e)
        new_csv.save(force_insert = True)

        raise
Exemple #13
0
def view_assignment(assignment_id):
    simple_archive_form = SimpleArchiveForm()

    # Convert the assignment in the URL into an ObjectId
    try:
        id = ObjectId(assignment_id)
    except InvalidId:
        logger.info("Invalid Assignment ID requested.")

        abort(404)

    # Retrieve the assignment
    try:
        assignment = Assignment.objects.get(id=id)
    except Assignment.DoesNotExist:
        logger.info("Non-extant ID requested.")

        abort(404)

    assignment.apply_personal_deadlines(current_user)

    # Get all of the submissions for this assignment
    submissions = list(Submission.objects(user=current_user.id, assignment=id).order_by("-most_recent", "-timestamp"))

    # Get student submissions if being viewed as teacher
    student_submissions = []
    students = []
    if current_user.account_type in ["teacher", "teaching_assistant"]:
        students = list(User.objects(classes=assignment.for_class, account_type="student").order_by("-email"))

        student_submissions = list(
            Submission.objects(user__in=[i.email for i in students], assignment=assignment_id, most_recent=True)
        )

    test_results = list(TestResult.objects(id__in=[i.test_results for i in submissions if i.test_results]))

    # Match test results to submissions
    for i in submissions:
        for j in test_results:
            if i.test_results == j.id:
                i.test_results_obj = j

    # Current time to be compared to submission test_request_timestamp
    now = datetime.datetime.now()

    # Flag to set refresh trigger if user is waiting on test results
    wait_and_refresh = False

    # Add the pretty version of each submissions timestamp
    for i in submissions:
        i.timestamp_pretty = pretty_time(i.timestamp)
        i.status = "Submitted"

        # If the user submitted a test request and there aren't any results
        if i.test_request_timestamp and not i.test_results:
            timedelta = now - i.test_request_timestamp
            i.show_resubmit = timedelta > config["STUDENT_RETRY_INTERVAL"]
            if not i.show_resubmit:
                i.status = "Waiting for test results..."
            else:
                i.status = "Test request timed out"
        elif i.test_results and i.test_results_obj.failed:
            i.status = "Tests Failed"
            i.show_resubmit = True
        elif i.test_results and not i.test_results_obj.failed:
            i.status = "Tests Completed"

    wait_and_refresh = any(i.status == "Waiting for test results..." for i in submissions)

    return render_template(
        "assignment.html",
        now=datetime.datetime.today(),
        create_time_element=create_time_element,
        assignment=assignment,
        submissions=submissions,
        simple_archive_form=simple_archive_form,
        wait_and_refresh=wait_and_refresh,
        new_submissions=[v for k, v in get_flashed_messages(with_categories=True) if k == "new_submission"],
        view_as_teacher=(current_user.account_type in ["teacher", "teaching_assistant"]),
        students=students,
        student_submissions=student_submissions,
        enumerate=enumerate,
    )
Exemple #14
0
def upload_submission(assignment_id):
    # Figure out which assignment the user asked for.
    try:
        id = ObjectId(assignment_id)
        assignment = Assignment.objects.get(id = id)
    except (InvalidId, Assignment.DoesNotExist) as e:
        logger.info("Could not retrieve assignment: %s", str(e))

        abort(404)

    # Figure out where we should redirect the user to once we're done.
    redirect_to = request.args.get("next") or request.referrer

    if not is_url_on_site(app, redirect_to):
        # Default going back to the assignment screen
        redirect_to = url_for(
            "view_assignment",
            assignment_id = assignment_id
        )

    assignment.apply_personal_deadlines(current_user)

    # Check if the assignment's cutoff date has passed
    if assignment.due_cutoff and \
            assignment.due_cutoff < datetime.datetime.today():
        logger.info("Submission rejected, cutoff date has already passed.")

        flash(
            "The cutoff date has already passed, your submission was not "
            "accepted.", category = "error"
        )

        return redirect(redirect_to)

    form = SimpleArchiveForm()
    if not form.validate_on_submit():
        logger.info(
            "Submission rejected due to internal validation problem."
        )

        flash(
            "Submission rejected due to internal validation problem. Try "
            "again.", category = "error"
        )

        return redirect(redirect_to)

    if not [i for i in form.archive.entries if i.data.filename]:
        logger.info("Submission rejected. User did not submit any files.")

        flash("You did not submit any files.", category = "error")

        return redirect(redirect_to)

    new_submission = Submission(
        assignment = id,
        user = current_user.id,
        timestamp = datetime.datetime.now(),
        test_type = "final" if form.marked_as_final.data else "public",
        most_recent = True
    )
    new_submission.id = ObjectId()

    logger.info(str(new_submission.to_dict()))

    # Craft a unique directory path where we will store the new submission. We
    # are guarenteed an ObjectId is unique. However we are not guarenteed that
    # we will have the proper permissions and that we will be able to make the
    # directory thus this could error because of that.
    new_submission.testables = new_submission.getFilePath()
    os.makedirs(new_submission.testables)

    # Save each file the user uploaded into the submissions directory
    for i in form.archive.entries:
        if not i.data.filename:
            continue

        #  Figure out where we want to save the user's file
        file_path = os.path.join(
            new_submission.testables, secure_filename(i.data.filename)
        )

        # Do the actual saving
        i.data.save(file_path)

    new_submission.uploaded_filenames.extend(
        secure_filename(i.data.filename) for i in form.archive.entries
            if i.data.filename
    )

    logger.info(
        "Succesfully uploaded a new submission (id = %s) with files %s.",
        str(new_submission.id),
        str(new_submission.uploaded_filenames)
    )

    # The old "most_recent" submission is no longer the most recent.
    Submission.objects(
        user = current_user.email,
        assignment = id,
        most_recent = True
    ).update(
        multi = False,
        unset__most_recent = 1
    )

    if assignment.test_harness:
        new_submission.test_request_timestamp = datetime.datetime.now()
        logger.info("Sent test request to shepherd for %s" % \
                        str(new_submission.id))

    new_submission.save()

    # Tell shepherd to start running tests if there is a test_harness.
    if assignment.test_harness:
        send_test_request(config["PUBLIC_SOCKET"], new_submission.id)

    # Communicate to the next page what submission was just added.
    flash(str(new_submission.id), category = "new_submission")

    flash(
        "Successfully uploaded %s %s." %
            (
                plural_if("file", len(new_submission.uploaded_filenames)),
                pretty_list(new_submission.uploaded_filenames)
            ),
        category = "message"
    )

    # Everything seems to have gone well
    return redirect(redirect_to)
Exemple #15
0
        return craft_response(
            error="The cutoff date has already passed, your submission was "
            "not accepted.")

    form = SimpleArchiveForm()
    if not form.validate_on_submit():
        flash("The files you passed in were invalid.", category="error")
        return redirect(redirect_to)

    if not [i for i in form.archive.entries if i.data.filename]:
        flash("You did not submit any files.", category="error")
        return redirect(redirect_to)

    new_submission = Submission(assignment=id,
                                user=current_user.id,
                                timestamp=datetime.datetime.now(),
                                marked_for_grading=True,
                                most_recent=True)
    new_submission.id = ObjectId()

    # Craft a unique directory path where we will store the new submission. We
    # are guarenteed an ObjectId is unique. However we are not guarenteed that
    # we will have the proper permissions and that we will be able to make the
    # directory thus this could error because of that.
    new_submission.testables = os.path.join(app.config["SUBMISSION_DIRECTORY"],
                                            str(new_submission.id))
    os.makedirs(new_submission.testables)

    # Save each file the user uploaded into the submissions directory
    for i in form.archive.entries:
        if not i.data.filename:
def _tar_bulk_submissions(archive_id, requester, assignment, email = ""):
    archive_id = ObjectId(archive_id)

    archive_file = temp_directory = ""

    # Find any expired archives and remove them
    deleted_files = []
    for i in Archive.objects(expires__lt = datetime.datetime.today()):
        deleted_files.append(i.file_location)

        if i.file_location:
            try:
                os.remove(i.file_location)
            except OSError as e:
                logger.warning(
                    "Could not remove expired archive at %s: %s.",
                    i.file_location, str(e)
                )

        i.delete()

    if deleted_files:
        logger.info("Deleted archives %s.", str(deleted_files))

    # This is the archive object we will eventually add to the database
    new_archive = Archive(
        id = archive_id,
        requester = requester,
        archive_type = "assignment_package"
    )

    temp_directory = archive_file = None
    try:
        # Form the query
        query = {"assignment": ObjectId(assignment)}

        # Only mention email in the query if it's not None or the empty
        # string, otherwise mongo will look for submissions that list the
        # user as None or the empty string (which should be exactly none of
        # the submission in the system).
        if email:
            query["user"] = email

        # Grab all the submissions
        submissions = list(Submission.objects(**query))

        if not submissions:
            logger.info("No submissions found matching query.")
            return

        # Organize all the submissions by user name, as this will closely
        # match the structure of the archive we will build.
        submission_map = {}
        for i in submissions:
            if i.user in submission_map:
                submission_map[i.user].append(i)
            else:
                submission_map[i.user] = [i]

        # Create a temporary directory we will create our archive in.
        temp_directory = tempfile.mkdtemp()

        # Create our directory tree. Instead of making new folders for each
        # submission and copying the user's files over however, we will
        # create symlinks to save space and time.
        for user, user_submissions in submission_map.items():
            # Create a directory for the user
            os.makedirs(os.path.join(temp_directory, user))

            # Create symlinks for all his submissions. Each symlink is
            # named after the submission date.
            for i in user_submissions:
                time_stamp = i.timestamp.strftime("%Y-%m-%d-%H-%M-%S")
                symlink_path = \
                    os.path.join(temp_directory, user, time_stamp)

                # In the highly unlikely event that two of the same user's
                # submissions have the same exact time stamp, we'll need to
                # add a marker to the end of the timestamp.
                marker = 0
                while os.path.exists(symlink_path +
                        ("-%d" % marker if marker > 0 else "")):
                    marker += 1

                if marker > 0:
                    symlink_path += "-%d" % marker

                original_path = i.getFilePath()

                # Detect if the submission's files are still on the filesystem
                if os.path.isdir(original_path):
                    # Create a symlink pointing to the actual submission
                    # directory with the name we gnerated
                    os.symlink(original_path, symlink_path)
                else:
                    # Create an empty text file marking the fact that a
                    # submissions existed but is no longer available.
                    open(symlink_path, "w").close()

        # Create the actual archive file.
        # TODO: Create it in galah's /var/ directory
        file_descriptor, archive_file = tempfile.mkstemp(suffix = ".tar.gz")
        os.close(file_descriptor)

        # Run tar and do the actual archiving. Will block until it's finished.
        subprocess.check_call(
            [
                "tar", "--dereference", "--create", "--gzip", "--directory",
                temp_directory, "--file", archive_file
            ] + submission_map.keys()
        )

        new_archive.file_location = archive_file

        new_archive.expires = \
            datetime.datetime.today() + config["TEACHER_ARCHIVE_LIFETIME"]

        new_archive.save(force_insert = True)
    except Exception as e:
        # If we created a temporary archive file we need to delete it.
        new_archive.file_location = None
        if archive_file:
            os.remove(archive_file)

        new_archive.error_string = str(e)
        new_archive.save(force_insert = True)

        raise
    finally:
        if temp_directory:
            shutil.rmtree(temp_directory)
def _zip_bulk_submissions(archive_id, requester, assignment, email = ""):
    archive_id = ObjectId(archive_id)

    archive_file = temp_directory = ""

    # Find any expired archives and remove them
    deleted_files = []
    for i in Archive.objects(expires__lt = datetime.datetime.today()):
        deleted_files.append(i.file_location)

        if i.file_location:
            try:
                os.remove(i.file_location)
            except OSError as e:
                logger.warning(
                    "Could not remove expired archive at %s: %s.",
                    i.file_location, str(e)
                )

        i.delete()

    if deleted_files:
        logger.info("Deleted archives %s.", str(deleted_files))

    # This is the archive object we will eventually add to the database
    new_archive = Archive(
        id = archive_id,
        requester = requester,
        archive_type = "assignment_package"
    )

    temp_directory = archive_file = None
    try:
        # Form the query
        query = {"assignment": ObjectId(assignment)}

        # Only mention email in the query if it's not None or the empty
        # string, otherwise mongo will look for submissions that list the
        # user as None or the empty string (which should be exactly none of
        # the submission in the system).
        if email:
            query["user"] = email
        else:
            # Otherwise, we need to be careful not to get teacher/TA submissions.
            assn = Assignment.objects.get(id = ObjectId(assignment))
            students = User.objects(
                account_type="student",
                classes = assn.for_class
            )
            query["user__in"] = [i.id for i in students]

        # Grab all the submissions
        submissions = list(Submission.objects(**query))

        if not submissions:
            logger.info("No submissions found matching query.")
            return

        # Organize all the submissions by user name, as this will closely
        # match the structure of the archive we will build.
        submission_map = {}
        for i in submissions:
            if i.user in submission_map:
                submission_map[i.user].append(i)
            else:
                submission_map[i.user] = [i]

        # Create a temporary directory we will create our archive in.
        temp_directory = tempfile.mkdtemp()

        # Create our directory tree. Instead of making new folders for each
        # submission and copying the user's files over however, we will
        # create symlinks to save space and time.
        for user, user_submissions in submission_map.items():
            # Create a directory for the user
            os.makedirs(os.path.join(temp_directory, user))

            # Create symlinks for all his submissions. Each symlink is
            # named after the submission date.
            for i in user_submissions:
                time_stamp = i.timestamp.strftime("%Y-%m-%d-%H-%M-%S")
                symlink_path = \
                    os.path.join(temp_directory, user, time_stamp)

                # In the highly unlikely event that two of the same user's
                # submissions have the same exact time stamp, we'll need to
                # add a marker to the end of the timestamp.
                marker = 0
                while os.path.exists(symlink_path +
                        ("-%d" % marker if marker > 0 else "")):
                    marker += 1

                if marker > 0:
                    symlink_path += "-%d" % marker

                original_path = i.getFilePath()

                # Detect if the submission's files are still on the filesystem
                if os.path.isdir(original_path):
                    # Create a symlink pointing to the actual submission
                    # directory with the name we gnerated
                    os.symlink(original_path, symlink_path)
                else:
                    # Create an empty text file marking the fact that a
                    # submissions existed but is no longer available.
                    open(symlink_path, "w").close()

        # Create the actual archive file.
        # TODO: Create it in galah's /var/ directory
        file_descriptor, archive_file = tempfile.mkstemp(suffix = ".zip")
        os.close(file_descriptor)

        # Run zip and do the actual archiving. Will block until it's finished.
        zipdir(temp_directory, archive_file)

        new_archive.file_location = archive_file

        new_archive.expires = \
            datetime.datetime.today() + config["TEACHER_ARCHIVE_LIFETIME"]

        new_archive.save(force_insert = True)
    except Exception as e:
        # If we created a temporary archive file we need to delete it.
        new_archive.file_location = None
        if archive_file:
            os.remove(archive_file)

        new_archive.error_string = str(e)
        new_archive.save(force_insert = True)

        raise
    finally:
        if temp_directory:
            shutil.rmtree(temp_directory)
def browse_assignments():
    # Grab all the current user's classes
    classes = Class.objects(id__in = current_user.classes).only("name")
    
    # Get the current time so we don't have to do it over and over again.
    now = datetime.datetime.today()

    if "show_all" in request.args:
        assignments = list(Assignment.objects(
            Q(for_class__in = current_user.classes) &
            (Q(hide_until = None) | Q(hide_until__lt = now))
        ).only("name", "due", "due_cutoff", "for_class"))
    else:
        assignments = list(Assignment.objects(
            Q(for_class__in = current_user.classes) &
            (Q(due__gt = now - datetime.timedelta(weeks = 1)) |
            Q(due_cutoff__gt = now - datetime.timedelta(weeks = 1))) &
            (Q(hide_until = None) | Q(hide_until__lt = now))
        ).only("name", "due", "due_cutoff", "for_class"))

    assignments = [i for i in assignments if
            not i.hide_until or i.hide_until < now]

    # Get the number of assignments that we could have gotten if we didn't
    # limit based on due date.
    all_assignments_count = Assignment.objects(
        Q(for_class__in = current_user.classes) &
        (Q(hide_until = None) | Q(hide_until__lt = now))
    ).count()

    submissions = list(Submission.objects(
        user = current_user.email,
        assignment__in = [i.id for i in assignments],
        most_recent = True
    ))

    # Add a property to all the assignments so the template can display their
    # respective class easier. Additionally, add a plain text version of the
    # due date
    for i in assignments:
        try:
            i.class_name = next((j.name for j in classes if j.id == i.for_class))
        except StopIteration:
            logger.error(
                "Assignment with id %s references non-existant class with id "
                "%s." % (str(i.id, i.for_class))
            )
            
            i.class_name = "DNE"

        # Figure out the status messages that we want to display to the user.
        submitted = next((j for j in submissions if j.assignment == i.id), None)
        i.status = i.status_color = None
        if submitted:
            i.status = (
                "You made a submission " +
                create_time_element(submitted.timestamp, now)
            )

            i.status_color = "#84B354"
        elif now < i.due:
            i.status = "You have not submitted yet"

            i.status_color = "#877150"
        elif now > i.due and i.due_cutoff and now > i.due_cutoff:
            i.status = "You have not submitted yet, and it is too late to do so"

            i.status_color = "#E9A400"
        elif now > i.due:
            i.status = "You have not submitted yet!"

            i.status_color = "#FB4313"

    # Sort the assignments by due_cutoff or due date if due_cutoff is not
    # assigned.
    assignments.sort(
        key = lambda i: i.due_cutoff if i.due_cutoff else i.due,
        reverse = True
    )

    return render_template(
        "assignments.html",
        assignments = assignments,
        hidden_assignments = -1 if "show_all" in request.args
                else all_assignments_count - len(assignments),
        create_time_element = create_time_element
    )
Exemple #19
0
def browse_assignments():
    # Grab all the current user's classes
    classes = Class.objects(id__in=current_user.classes).only("name")

    # Get the current time so we don't have to do it over and over again.
    now = datetime.datetime.today()

    if "show_all" in request.args:
        assignments = list(
            Assignment.objects(
                Q(for_class__in=current_user.classes)
                & (Q(hide_until=None) | Q(hide_until__lt=now))).only(
                    "name", "due", "due_cutoff", "for_class"))
    else:
        assignments = list(
            Assignment.objects(
                Q(for_class__in=current_user.classes)
                & (Q(due__gt=now - datetime.timedelta(weeks=1))
                   | Q(due_cutoff__gt=now - datetime.timedelta(weeks=1)))
                & (Q(hide_until=None) | Q(hide_until__lt=now))).only(
                    "name", "due", "due_cutoff", "for_class"))

    assignments = [
        i for i in assignments if not i.hide_until or i.hide_until < now
    ]

    # Get the number of assignments that we could have gotten if we didn't
    # limit based on due date.
    all_assignments_count = Assignment.objects(
        Q(for_class__in=current_user.classes)
        & (Q(hide_until=None) | Q(hide_until__lt=now))).count()

    submissions = list(
        Submission.objects(assignment__in=[i.id for i in assignments],
                           most_recent=True))

    # Add a property to all the assignments so the template can display their
    # respective class easier. Additionally, add a plain text version of the
    # due date
    for i in assignments:
        try:
            i.class_name = next(
                (j.name for j in classes if j.id == i.for_class))
        except StopIteration:
            app.logger.error("Assignment with id %s references non-existant "
                             "class with id %s." % (str(i.id, i.for_class)))

            i.class_name = "DNE"

        # Figure out the status messages that we want to display to the user.
        submitted = next((j for j in submissions if j.assignment == i.id),
                         None)
        i.status = i.status_color = None
        if submitted:
            i.status = ("You made a submission " +
                        create_time_element(submitted.timestamp, now))

            i.status_color = "#84B354"
        elif now < i.due:
            i.status = "You have not submitted yet"

            i.status_color = "#877150"
        elif now > i.due and i.due_cutoff and now > i.due_cutoff:
            i.status = "You have not submitted yet, and it is too late to do so"

            i.status_color = "#E9A400"
        elif now > i.due:
            i.status = "You have not submitted yet!"

            i.status_color = "#FB4313"

    # Sort the assignments by due_cutoff or due date if due_cutoff is not
    # assigned.
    assignments.sort(key=lambda i: i.due_cutoff if i.due_cutoff else i.due,
                     reverse=True)

    return render_template("assignments.html",
                           assignments=assignments,
                           hidden_assignments=-1 if "show_all" in request.args
                           else all_assignments_count - len(assignments),
                           create_time_element=create_time_element)
Exemple #20
0
def _tar_bulk_submissions(archive_id, requester, assignment, email = ""):
    archive_id = ObjectId(archive_id)

    archive_file = temp_directory = ""

    # Find any expired archives and remove them
    for i in Archive.objects(expires__lt = datetime.datetime.today()):
        if i.file_location:
            logger.debug("Erasing old archive at '%s'." % i.file_location)

            try:
                os.remove(i.file_location)
            except OSError:
                logger.warn(
                    "Could not remove expired archive at %s.",
                    i.file_location
                )

        i.delete()

    # This is the archive object we will eventually add to the database
    new_archive = Archive(
        id = archive_id,
        requester = requester,
        archive_type = "assignment_package"
    )

    temp_directory = archive_file = None
    try:
        # Form the query
        query = {"assignment": ObjectId(assignment)}

        # Only mention email in the query if it's not None or the empty
        # string, otherwise mongo will look for submissions that list the
        # user as None or the empty string (which should be exactly none of
        # the submission in the system).
        if email:
            query["user"] = email

        # Grab all the submissions
        submissions = list(Submission.objects(**query))

        if not submissions:
            raise RuntimeError("No submissions found matching query.")

        # Organize all the submissions by user name, as this will closely
        # match the structure of the archive we will build.
        submission_map = {}
        for i in submissions:
            if i.user in submission_map:
                submission_map[i.user].append(i)
            else:
                submission_map[i.user] = [i]

        # Create a temporary directory we will create our archive in.
        temp_directory = tempfile.mkdtemp()

        # Create our directory tree. Instead of making new folders for each
        # submission and copying the user's files over however, we will
        # create symlinks to save space and time.
        for user, user_submissions in submission_map.items():
            # Create a directory for the user
            os.makedirs(os.path.join(temp_directory, user))

            # Create symlinks for all his submissions. Each symlink is
            # named after the submission date.
            for i in user_submissions:
                time_stamp = i.timestamp.strftime("%Y-%m-%d-%H-%M-%S")
                symlink_path = \
                    os.path.join(temp_directory, user, time_stamp)

                # In the highly unlikely event that two of the same user's
                # submissions have the same exact time stamp, we'll need to
                # add a marker to the end of the timestamp.
                marker = 0
                while os.path.isfile(symlink_path +
                        ("-%d" % marker if marker > 0 else "")):
                    marker += 1

                if marker > 0:
                    symlink_path += "-%d" % marker

                original_path = \
                    os.path.join(config["SUBMISSION_DIRECTORY"], str(i.id))

                # Detect if the submission's files are still on the filesystem
                if os.path.isdir(original_path):
                    # Create a symlink pointing to the actual submission
                    # directory with the name we gnerated
                    os.symlink(original_path, symlink_path)
                else:
                    # Create an empty text file marking the fact that a
                    # submissions existed but is no longer available.
                    open(symlink_path, "w").close()

        # Create the actual archive file.
        # TODO: Create it in galah's /var/ directory
        file_descriptor, archive_file = tempfile.mkstemp(suffix = ".tar.gz")
        os.close(file_descriptor)

        # Run tar and do the actual archiving. Will block until it's finished.
        subprocess.check_call(
            [
                "tar", "--dereference", "--create", "--gzip", "--directory",
                temp_directory, "--file", archive_file
            ] + submission_map.keys()
        )

        new_archive.file_location = archive_file

        new_archive.expires = \
            datetime.datetime.today() + datetime.timedelta(hours = 2)

        new_archive.save(force_insert = True)
    except Exception as e:
        logger.exception("An error occured while creating an archive.")

        # If we created a temporary archive file we need to delete it.
        new_archive.file_location = None
        if archive_file:
            os.remove(archive_file)

        new_archive.error_string = str(e)
        new_archive.save(force_insert = True)
    finally:
        if temp_directory:
            shutil.rmtree(temp_directory)
Exemple #21
0
def upload_submission(assignment_id):
    # Figure out which assignment the user asked for.
    try:
        id = ObjectId(assignment_id)
        assignment = Assignment.objects.get(id=id)
    except (InvalidId, Assignment.DoesNotExist) as e:
        logger.info("Could not retrieve assignment: %s", str(e))

        abort(404)

    # Figure out where we should redirect the user to once we're done.
    redirect_to = request.args.get("next") or request.referrer

    if not is_url_on_site(app, redirect_to):
        # Default going back to the assignment screen
        redirect_to = url_for("view_assignment", assignment_id=assignment_id)

    assignment.apply_personal_deadlines(current_user)

    # Check if the assignment's cutoff date has passed
    if assignment.due_cutoff and \
            assignment.due_cutoff < datetime.datetime.today():
        logger.info("Submission rejected, cutoff date has already passed.")

        flash(
            "The cutoff date has already passed, your submission was not "
            "accepted.",
            category="error")

        return redirect(redirect_to)

    form = SimpleArchiveForm()
    if not form.validate_on_submit():
        logger.info("Submission rejected due to internal validation problem.")

        flash(
            "Submission rejected due to internal validation problem. Try "
            "again.",
            category="error")

        return redirect(redirect_to)

    if not [i for i in form.archive.entries if i.data.filename]:
        logger.info("Submission rejected. User did not submit any files.")

        flash("You did not submit any files.", category="error")

        return redirect(redirect_to)

    new_submission = Submission(
        assignment=id,
        user=current_user.id,
        timestamp=datetime.datetime.now(),
        test_type="final" if form.marked_as_final.data else "public",
        most_recent=True)
    new_submission.id = ObjectId()

    logger.info(str(new_submission.to_dict()))

    # Craft a unique directory path where we will store the new submission. We
    # are guarenteed an ObjectId is unique. However we are not guarenteed that
    # we will have the proper permissions and that we will be able to make the
    # directory thus this could error because of that.
    new_submission.testables = new_submission.getFilePath()
    os.makedirs(new_submission.testables)

    # Save each file the user uploaded into the submissions directory
    for i in form.archive.entries:
        if not i.data.filename:
            continue

        #  Figure out where we want to save the user's file
        file_path = os.path.join(new_submission.testables,
                                 secure_filename(i.data.filename))

        # Do the actual saving
        i.data.save(file_path)

    new_submission.uploaded_filenames.extend(
        secure_filename(i.data.filename) for i in form.archive.entries
        if i.data.filename)

    logger.info(
        "Succesfully uploaded a new submission (id = %s) with files %s.",
        str(new_submission.id), str(new_submission.uploaded_filenames))

    # The old "most_recent" submission is no longer the most recent.
    Submission.objects(user=current_user.email,
                       assignment=id,
                       most_recent=True).update(multi=False,
                                                unset__most_recent=1)

    if assignment.test_harness:
        new_submission.test_request_timestamp = datetime.datetime.now()
        logger.info("Sent test request to shepherd for %s" % \
                        str(new_submission.id))

    new_submission.save()

    # Tell shepherd to start running tests if there is a test_harness.
    if assignment.test_harness:
        send_test_request(config["PUBLIC_SOCKET"], new_submission.id)

    # Communicate to the next page what submission was just added.
    flash(str(new_submission.id), category="new_submission")

    flash("Successfully uploaded %s %s." %
          (plural_if("file", len(new_submission.uploaded_filenames)),
           pretty_list(new_submission.uploaded_filenames)),
          category="message")

    # Everything seems to have gone well
    return redirect(redirect_to)
Exemple #22
0
def _create_assignment_csv(csv_id, requester, assignment):
    csv_id = ObjectId(csv_id)

    csv_file = temp_directory = ""

    # Find any expired archives and remove them
    deleted_files = []
    for i in CSV.objects(expires__lt=datetime.datetime.today()):
        deleted_files.append(i.file_location)

        if i.file_location:
            try:
                os.remove(i.file_location)
            except OSError as e:
                logger.warning("Could not remove expired csv file at %s: %s.",
                               i.file_location, str(e))

        i.delete()

    if deleted_files:
        logger.info("Deleted csv files %s.", str(deleted_files))

    # This is the CSV object that will be added to the database
    new_csv = CSV(id=csv_id, requester=requester)

    temp_directory = csv_file = None
    try:
        assn = Assignment.objects.get(id=ObjectId(assignment))

        # Grab all student users for this class.
        users = list(
            User.objects(account_type="student", classes=assn.for_class))

        # Form the query
        query = {
            "assignment": ObjectId(assignment),
            "most_recent": True,
            "user__in": [i.id for i in users]
        }

        # Grab the most recent submissions from each user.
        submissions = list(Submission.objects(**query))

        # Create the actual csv file.
        csv_file = open(os.path.join(config["CSV_DIRECTORY"], str(csv_id)),
                        "w")

        for i in submissions:
            score = "None"
            if i.test_results:
                test_result = TestResult.objects.get(id=i.test_results)
                score = str(test_result.score)

            print >> csv_file, "%s,%s,%s" % \
                (i.user, score, i.timestamp.strftime("%Y-%m-%d-%H-%M-%S"))

        csv_file.close()

        new_csv.file_location = os.path.join(config["CSV_DIRECTORY"],
                                             str(csv_id))

        new_csv.expires = \
            datetime.datetime.today() + config["TEACHER_CSV_LIFETIME"]

        new_csv.save(force_insert=True)
    except Exception as e:
        new_csv.file_location = None
        os.remove(os.path.join(config["CSV_DIRECTORY"], str(csv_id)))

        new_csv.error_string = str(e)
        new_csv.save(force_insert=True)

        raise
Exemple #23
0
def _create_gradebook_csv(csv_id, requester, class_id, fill=0):
    csv_id = ObjectId(csv_id)

    csv_file = temp_directory = ""

    # Find any expired archives and remove them
    deleted_files = []
    for i in CSV.objects(expires__lt = datetime.datetime.today()):
        deleted_files.append(i.file_location)

        if i.file_location:
            try:
                os.remove(i.file_location)
            except OSError as e:
                logger.warning(
                    "Could not remove expired csv file at %s: %s.",
                    i.file_location, str(e)
                )

        i.delete()

    if deleted_files:
        logger.info("Deleted csv files %s.", str(deleted_files))

    # This is the CSV object that will be added to the database
    new_csv = CSV(
        id = csv_id,
        requester = requester
    )

    temp_directory = csv_file = None
    try:
        # Create the actual csv file.
        csv_file = open(os.path.join(config["CSV_DIRECTORY"], str(csv_id)), "w")

        the_class = Class.objects.get(id = ObjectId(class_id))

        # Grab all assignments in this class
        assns = list(
            Assignment.objects(for_class = the_class.id)
        )

        print >> csv_file, "%s,%s" % \
            ("Username", ",".join('"{0}"'.format(i.name) for i in assns))

        # Grab all student users for this class.
        users = list(
            User.objects(
                account_type = "student",
                classes = the_class.id
            )
        )

        assn_ids = [i.id for i in assns]
        for user in users:
            # Query for user's most recent submissions in the known assignments
            query = {
                "assignment__in": assn_ids,
                "most_recent": True,
                "user": user.id
            }

            submissions = list(Submission.objects(**query))

            # Initialize each assignment score to empty at first.
            assn_to_score = OrderedDict((i, str(fill)) for i in assn_ids)

            # Go through submissions, associating scores with assignment
            for sub in submissions:
                if sub.test_results:
                    test_result = TestResult.objects.get(id = sub.test_results)
                    if test_result.score is not None:
                        assn_to_score[sub.assignment] = str(test_result.score)

            # Write gradebook results to csv file.
            print >> csv_file, "%s,%s" % \
                (user.email, ",".join(assn_to_score.values()))

        csv_file.close()

        new_csv.file_location = os.path.join(config["CSV_DIRECTORY"], str(csv_id))

        new_csv.expires = \
            datetime.datetime.today() + config["TEACHER_CSV_LIFETIME"]

        new_csv.save(force_insert = True)
    except Exception as e:
        new_csv.file_location = None
        os.remove(os.path.join(config["CSV_DIRECTORY"], str(csv_id)))

        new_csv.error_string = str(e)
        new_csv.save(force_insert = True)

        raise