Пример #1
0
    def POST(self, courseid, groupid):
        """ Edit a group """
        course, _ = get_course_and_check_rights(courseid, allow_all_staff=False)
        student_list, tutor_list = self.get_user_lists(course, groupid)
        group = get_database().groups.find_one({"_id": ObjectId(groupid), "course_id": courseid})

        error = ""
        try:
            data = web.input(group_tutor=[], group_student=[])
            if "delete" in data:
                get_database().groups.remove({"_id": ObjectId(groupid)})
                raise web.seeother("/admin/"+courseid+"/students")
            else:
                data["group_tutor"] = [tutor for tutor in data["group_tutor"] if tutor in tutor_list]
                data["group_student"] = [student for student in data["group_student"] if student in student_list]

                if len(data["group_student"]) > int(data['size']):
                    error = 'Too many students for given group size.'
                elif data['description']:
                    group = get_database().groups.find_one_and_update(
                        {"_id": ObjectId(groupid)},
                        {"$set": {"description": data["description"],
                                  "users": data["group_student"], "tutors": data["group_tutor"],
                                  "size": abs(int(data["size"]))}}, return_document=ReturnDocument.AFTER)
                else:
                    error = 'No group description given.'
        except:
            error = 'User returned an invalid form.'

        return renderer.course_admin.edit_group(course, student_list, tutor_list, group, error, True)
def init_backend_interface(plugin_manager):
    """ inits everything that makes the backend working """

    # Updates the submissions that have a jobid with the status error, as the server restarted """
    get_database().submissions.update({'jobid': {"$exists": True}},
                                      {"$unset": {'jobid': ""},
                                       "$set": {'status': 'error', 'grade': 0.0, 'text': 'Internal error. Server restarted'}}, multi=True)

    # Create the job manager
    get_job_manager.job_manager = JobManager(
        INGIniousConfiguration["docker_instances"],
        INGIniousConfiguration["containers"],
        INGIniousConfiguration["tasks_directory"],
        INGIniousConfiguration.get(
            "callback_managers_threads",
            1),
        INGIniousConfiguration.get(
            "slow_pool_size",
            4),
        INGIniousConfiguration.get(
            "fast_pool_size",
            4),
        INGIniousConfiguration.get(
            "containers_hard",
            []),
        plugin_manager)
Пример #3
0
def _job_done_callback(submissionid, task, job):
    """ Callback called by JobManager when a job is done. Updates the submission in the database with the data returned after the completion of the job """
    submission = get_submission(submissionid, False)
    submission = get_input_from_submission(submission)

    job = _parse_text(task, job)

    data = {
        "status": ("done" if job["result"] == "success" or job["result"] == "failed" else "error"),  # error only if error was made by INGInious
        "result": job["result"],
        "grade": job["grade"],
        "text": job.get("text", None),
        "tests": job.get("tests", None),
        "problems": (job["problems"] if "problems" in job else {}),
        "archive": (get_gridfs().put(base64.b64decode(job["archive"])) if "archive" in job else None)
    }

    # Store additional data
    dont_dump = ["task", "course", "input"]
    for index in job:
        if index not in data and index not in dont_dump:
            data[index] = job[index]

    # Save submission to database
    get_database().submissions.update(
        {"_id": submission["_id"]},
        {"$set": data}
    )

    for username in submission["username"]:
        UserData(username).update_stats(submission, job)

    PluginManager.get_instance().call_hook("submission_done", submission=submission, job=job)
Пример #4
0
def add_job(task, inputdata, debug=False):
    """ Add a job in the queue and returns a submission id.
        task is a Task instance and inputdata is the input as a dictionary
        If debug is true, more debug data will be saved
    """
    if not User.is_logged_in():
        raise Exception("A user must be logged in to submit an object")

    username = User.get_username()
    course = FrontendCourse(task.get_course_id())

    obj = {
        "courseid": task.get_course_id(),
        "taskid": task.get_id(),
        "input": get_gridfs().put(
            json.dumps(inputdata)),
        "status": "waiting",
        "submitted_on": datetime.now()}

    if course.is_group_course() and username not in course.get_staff(True):
        group = get_database().groups.find_one({"course_id": task.get_course_id(), "users": username})
        obj.update({"username": group["users"]})
    else:
        obj.update({"username": [username]})

    submissionid = get_database().submissions.insert(obj)

    PluginManager.get_instance().call_hook("new_submission", submissionid=submissionid, submission=obj, inputdata=inputdata)

    get_job_manager().new_job(task, inputdata, (lambda job: _job_done_callback(submissionid, task, job)), "Frontend - {}".format(username), debug)

    return submissionid
Пример #5
0
    def is_user_registered(self, username, check_group=False):
        """ Returns True if the user is registered """
        has_group = (not check_group) or \
                    (get_database().groups.find_one({"users": username, "course_id": self.get_id()}) is not None)

        return (get_database().registration.find_one({"username": username, "courseid": self.get_id()}) is not None)\
               and has_group or username in self.get_staff()
Пример #6
0
def drop_batch_job(batch_job_id):
    """ Delete a **finished** batch job from the database """
    job = get_database().batch_jobs.find_one({"_id": ObjectId(batch_job_id)})
    if "result" not in job:
        raise Exception("Batch job is still running, cannot delete it")
    get_database().batch_jobs.remove({"_id": ObjectId(batch_job_id)})
    if "file" in job["result"]:
        get_gridfs().delete(job["result"]["file"])
Пример #7
0
 def register_user(self, username, password=None, force=False):
     """ Register a user to the course. Returns True if the registration succeeded, False else. """
     if not force:
         if not self.is_registration_possible(username):
             return False
         if self.is_password_needed_for_registration() and self._registration_password != password:
             return False
     if self.is_open_to_user(username):
         return False  # already registered?
     get_database().registration.insert({"username": username, "courseid": self.get_id(), "date": datetime.now()})
     return True
Пример #8
0
 def view_task(self, courseid, taskid):
     """ Set in the database that the user has viewed this task """
     # Insert a new entry if no one exists
     get_database().user_tasks.update({"username": self.username,
                                       "courseid": courseid,
                                       "taskid": taskid},
                                      {"$setOnInsert": {"username": self.username,
                                                        "courseid": courseid,
                                                        "taskid": taskid,
                                                        "tried": 0,
                                                        "succeeded": False,
                                                        "grade": 0.0}},
                                      upsert=True)
Пример #9
0
def init(plugin_manager):
    """ inits everything that makes the backend working """

    # Updates the submissions that are waiting with the status error, as the server restarted
    get_database().submissions.update(
        {"status": "waiting"},
        {
            "$unset": {"jobid": ""},
            "$set": {"status": "error", "grade": 0.0, "text": "Internal error. Server restarted"},
        },
        multi=True,
    )

    # Updates all batch job still running
    get_database().batch_jobs.update(
        {"result": {"$exists": False}},
        {"$set": {"result": {"retval": -1, "stderr": "Internal error. Server restarted"}}},
        multi=True,
    )

    # Create the job manager
    backend_type = INGIniousConfiguration.get("backend", "local")
    if backend_type == "local":
        get_job_manager.job_manager = LocalJobManager(
            INGIniousConfiguration.get(
                "containers", {"default": "ingi/inginious-c-default", "sekexe": "ingi/inginious-c-sekexe"}
            ),
            INGIniousConfiguration.get("local_agent_tmp_dir", "/tmp/inginious_agent"),
            plugin_manager,
        )
    elif backend_type == "remote":
        get_job_manager.job_manager = RemoteDockerJobManager(
            INGIniousConfiguration.get("docker_daemons", []),
            INGIniousConfiguration.get(
                "containers", {"default": "ingi/inginious-c-default", "sekexe": "ingi/inginious-c-sekexe"}
            ),
            plugin_manager,
        )
    elif backend_type == "remote_manual":
        get_job_manager.job_manager = RemoteManualAgentJobManager(
            INGIniousConfiguration.get("agents", [{"host": "localhost", "port": 5001}]),
            INGIniousConfiguration.get(
                "containers", {"default": "ingi/inginious-c-default", "sekexe": "ingi/inginious-c-sekexe"}
            ),
            plugin_manager,
        )
    else:
        raise Exception("Unknown backend {}".format(backend_type))
def get_user_submissions(task):
    """ Get all the user's submissions for a given task """
    if not User.is_logged_in():
        raise Exception("A user must be logged in to get his submissions")
    cursor = get_database().submissions.find({"username": User.get_username(), "taskid": task.get_id(), "courseid": task.get_course_id()})
    cursor.sort([("submitted_on", -1)])
    return list(cursor)
def add_job(task, inputdata, debug=False):
    """ Add a job in the queue and returns a submission id.
        task is a Task instance and inputdata is the input as a dictionary
        If debug is true, more debug data will be saved
    """
    if not User.is_logged_in():
        raise Exception("A user must be logged in to submit an object")

    username = User.get_username()

    jobid = get_job_manager().new_job_id()

    obj = {
        "username": username,
        "courseid": task.get_course_id(),
        "taskid": task.get_id(),
        "input": get_gridfs().put(
            json.dumps(inputdata)),
        "status": "waiting",
        "jobid": jobid,
        "submitted_on": datetime.now()}
    submissionid = get_database().submissions.insert(obj)

    PluginManager.get_instance().call_hook("new_submission", submissionid=submissionid, submission=obj, jobid=jobid, inputdata=inputdata)

    get_job_manager().new_job(task, inputdata, job_done_callback, "Frontend - {}".format(username), jobid, debug)

    return submissionid
Пример #12
0
 def page(self, course, task):
     """ Get all data and display the page """
     data = list(get_database().user_tasks.find({"courseid": course.get_id(), "taskid": task.get_id(), "username": {"$in": course.get_registered_users()}}))
     data = [dict(f.items() + [("url", self.submission_url_generator(course, task, f))]) for f in data]
     if "csv" in web.input():
         return make_csv(data)
     return renderer.course_admin.task_info(course, task, data)
Пример #13
0
 def download_course(self, course, include_old_submissions=False):
     """ Download all submissions for a course """
     submissions = list(get_database().submissions.find(
         {"courseid": course.get_id(), "username": {"$in": course.get_registered_users()}, "status": {"$in": ["done", "error"]}}))
     if not include_old_submissions:
         submissions = self._keep_best_submission(submissions)
     return self.download_submission_set(submissions, '_'.join([course.get_id()]) + '.tgz', ['username', 'taskid'])
Пример #14
0
 def download_student_task(self, course, username, taskid, include_old_submissions=True):
     """ Download all submissions for a user for given task """
     submissions = list(get_database().submissions.find(
         {"username": username, "courseid": course.get_id(), "taskid": taskid, "status": {"$in": ["done", "error"]}}))
     if not include_old_submissions:
         submissions = self._keep_best_submission(submissions)
     return self.download_submission_set(submissions, '_'.join([username, course.get_id(), taskid]) + '.tgz', [])
Пример #15
0
 def page(self, course, username, task):
     """ Get all data and display the page """
     data = list(get_database().submissions.find({"username": username, "courseid": course.get_id(), "taskid": task.get_id()}).sort([("submitted_on", pymongo.DESCENDING)]))
     data = [dict(f.items() + [("url", self.submission_url_generator(course, str(f["_id"])))]) for f in data]
     if "csv" in web.input():
         return make_csv(data)
     return renderer.course_admin.student_task(course, username, task, data)
Пример #16
0
 def get_registered_users(self, with_admins=True):
     """ Get all the usernames that are registered to this course (in no particular order)"""
     l = [entry['username'] for entry in list(get_database().registration.find({"courseid": self.get_id()}, {"username": True, "_id": False}))]
     if with_admins:
         return list(set(l + self.get_staff()))
     else:
         return l
Пример #17
0
    def POST(self, courseid):
        """ POST request """
        course, _ = get_course_and_check_rights(courseid)

        if not course.is_group_course():
            raise web.notfound()

        error = ""
        try:
            data = web.input()
            if not data['group_description']:
                error = 'No group description given.'
            else:
                get_database().groups.insert({"course_id": courseid, "users": [], "tutors": [], "size": 2,
                                              "description": data['group_description']})
        except:
            error = 'User returned an invalid form.'

        return self.page(course, error, True)
Пример #18
0
    def GET(self, courseid, groupid):
        """ Edit a group """
        course, _ = get_course_and_check_rights(courseid, allow_all_staff=False)
        student_list, tutor_list = self.get_user_lists(course, groupid)

        group = get_database().groups.find_one({"_id": ObjectId(groupid), "course_id": courseid})

        if group:
            return renderer.course_admin.edit_group(course, student_list, tutor_list, group, "", False)
        else:
            raise web.notfound()
def get_user_last_submissions(query, limit, one_per_task=False):
    """ Get last submissions of a user """
    if not User.is_logged_in():
        raise Exception("A user must be logged in to get his submissions")
    request = query.copy()
    request.update({"username": User.get_username()})

    # We only want the last x task tried, modify the request
    if one_per_task is True:
        data = get_database().submissions.aggregate([
            {"$match": request},
            {"$sort": {"submitted_on": pymongo.DESCENDING}},
            {"$group": {"_id": {"courseid": "$courseid", "taskid": "$taskid"}, "orig_id": {"$first": "$_id"}, "submitted_on": {"$first": "$submitted_on"}}},
            {"$sort": {"submitted_on": pymongo.DESCENDING}},
            {"$limit": limit}
        ])
        request = {"_id": {"$in": [d["orig_id"] for d in list(data)]}}

    cursor = get_database().submissions.find(request)
    cursor.sort([("submitted_on", -1)]).limit(limit)
    return list(cursor)
Пример #20
0
def _batch_job_done_callback(batch_job_id, result):
    """ Called when the batch job with id jobid has finished.
        result is a dictionnary, containing:

        - {"retval": 0, "stdout": "...", "stderr": "...", "file": "..."}
            if everything went well.(where file is a tgz file containing the content of the / output folder from the container)
        - {"retval": "...", "stdout": "...", "stderr": "..."}
            if the container crashed (retval is an int != 0)
        - {"retval": -1, "stderr": "the error message"}
            if the container failed to start
    """

    # If there is a tgz file to save, put it in gridfs
    if "file" in result:
        result["file"] = get_gridfs().put(result["file"].read())

    # Save submission to database
    get_database().batch_jobs.update(
        {"_id": batch_job_id},
        {"$set": {"result": result}}
    )
Пример #21
0
def init(app, session_test=None):
    """ 
        Init the session. Should be call before starting the web.py server
        session_test is specified to emulate a session (used for tests)
    """
    if session_test is None:
        if web.config.get('_session') is None:
            get_session.session = web.session.Session(app, MongoStore(get_database(), 'sessions'))
            web.config._session = get_session.session  # pylint: disable=protected-access
        else:
            get_session.session = web.config._session  # pylint: disable=protected-access
    else:
        get_session.session = AttrDict(copy.deepcopy(session_test))
Пример #22
0
    def page(self, course, task):
        """ Get all data and display the page """

        groups = []
        if course.is_group_course():
            groups = get_database().groups.find({"course_id": course.get_id()})
        groups = sorted(groups, key=lambda item: item["description"])

        groups.insert(0, {"_id": 0, "users": course.get_staff(), "description": "Course staff", "tutors": {}})

        results = list(get_database().user_tasks.find({"courseid": course.get_id(), "taskid": task.get_id(),
                                                  "username": {"$in": course.get_registered_users()}}))
        data = {}
        for user in results:
            user["url"] = self.submission_url_generator(course, task, user)
            data[user["username"]] = user

        user_csv = [dict(data[key].items()) for key in data.keys()]
        if "csv" in web.input():
            return make_csv(user_csv)

        return renderer.course_admin.task_info(course, task, data, groups)
Пример #23
0
def get_all_batch_jobs_for_course(course_id):
    """ Returns all the batch jobs for the course course id. Batch jobs are dicts in the form
        {"courseid": "...", "container_name": "...", "submitted_on":"..."} if the job is still ongoing, and
        {"courseid": "...", "container_name": "...", "submitted_on":"...", "results": {}} if the job is done.
        the dict result can be either:

        - {"retval":0, "stdout": "...", "stderr":"...", "file":"..."}
            if everything went well. (file is an gridfs id to a tgz file)
        - {"retval":"...", "stdout": "...", "stderr":"..."}
            if the container crashed (retval is an int != 0)
        - {"retval":-1, "stderr": "the error message"}
            if the container failed to start
    """
    return list(get_database().batch_jobs.find({"courseid": course_id}))
Пример #24
0
    def get_user_lists(self, course, groupid):
        """ Get the available student and tutor lists for group edition"""
        student_list, tutor_list = course.get_registered_users(False), course.get_staff(False)

        # Remove grouped users from the accessible list for the group
        grouped_users = get_database().groups.aggregate([
            {"$match": {"_id": {"$ne": ObjectId(groupid)}}},
            {"$group": {"_id": "$_id", "gusers": {"$addToSet": "$users"}}}])

        for result in grouped_users:
            for users in result["gusers"]:
                student_list = [student for student in student_list if student not in users]

        return student_list, tutor_list
Пример #25
0
def get_batch_job_status(batch_job_id):
    """ Returns the batch job with id batch_job_id Batch jobs are dicts in the form
        {"courseid": "...", "container_name": "..."} if the job is still ongoing, and
        {"courseid": "...", "container_name": "...", "results": {}} if the job is done.
        the dict result can be either:

        - {"retval":0, "stdout": "...", "stderr":"...", "file":"..."}
            if everything went well. (file is an gridfs id to a tgz file)
        - {"retval":"...", "stdout": "...", "stderr":"..."}
            if the container crashed (retval is an int != 0)
        - {"retval":-1, "stderr": "the error message"}
            if the container failed to start
    """
    return get_database().batch_jobs.find_one({"_id": ObjectId(batch_job_id)})
Пример #26
0
    def page(self, course):
        """ Get all data and display the page """
        data = list(get_database().user_tasks.aggregate(
            [
                {
                    "$match":
                        {
                            "courseid": course.get_id(),
                            "username": {"$in": course.get_registered_users()}
                        }
                },
                {
                    "$group":
                        {
                            "_id": "$taskid",
                            "viewed": {"$sum": 1},
                            "attempted": {"$sum": {"$cond": [{"$ne": ["$tried", 0]}, 1, 0]}},
                            "attempts": {"$sum": "$tried"},
                            "succeeded": {"$sum": {"$cond": ["$succeeded", 1, 0]}}
                        }
                }
            ]))

        # Load tasks and verify exceptions
        files = get_readable_tasks(course.get_id())
        output = {}
        errors = []
        for task in files:
            try:
                output[task] = course.get_task(task)
            except Exception as inst:
                errors.append({"taskid": task, "error": str(inst)})
        tasks = OrderedDict(sorted(output.items(), key=lambda t: t[1].get_order()))

        # Now load additionnal informations
        result = OrderedDict()
        for taskid in tasks:
            result[taskid] = {"name": tasks[taskid].get_name(), "viewed": 0, "attempted": 0, "attempts": 0, "succeeded": 0,
                              "url": self.submission_url_generator(course, taskid)}
        for entry in data:
            if entry["_id"] in result:
                result[entry["_id"]]["viewed"] = entry["viewed"]
                result[entry["_id"]]["attempted"] = entry["attempted"]
                result[entry["_id"]]["attempts"] = entry["attempts"]
                result[entry["_id"]]["succeeded"] = entry["succeeded"]
        if "csv" in web.input():
            return make_csv(result)
        return renderer.course_admin.task_list(course, result, errors)
Пример #27
0
    def get_course_data_for_users(cls, courseid, users=None):
        """
            Returns data of users for a specific course. users is a list of username. If users is none, data from all users will be returned.

            The returned value is a dict:

            {"username": {"task_tried": 0, "total_tries": 0, "task_succeeded": 0, "task_grades":{"task_1": 100.0, "task_2": 0.0, ...}}}

            Please note that only the task already seen at least one time will be present in the dict task_grades.
        """
        from frontend.custom.courses import FrontendCourse

        course = FrontendCourse(courseid)
        match = {"courseid": courseid}
        if users is not None:
            match["username"] = {"$in": users}

        tasks = course.get_tasks()
        taskids = tasks.keys()
        match["taskid"] = {"$in": taskids}

        data = list(get_database().user_tasks.aggregate(
            [
                {"$match": match},
                {"$group": {
                    "_id": "$username",
                    "task_tried": {"$sum": {"$cond": [{"$ne": ["$tried", 0]}, 1, 0]}},
                    "total_tries": {"$sum": "$tried"},
                    "task_succeeded": {"$addToSet": {"$cond": ["$succeeded", "$taskid", False]}},
                    "task_grades": {"$addToSet": {"taskid": "$taskid", "grade": "$grade"}}
                }}
            ]))
        if len(data) != 0:
            return_data = {}
            for result in data:
                username = result["_id"]
                user_tasks = set([taskid for taskid, task in tasks.iteritems() if task.is_visible_by_user(username)])
                result["total_tasks"] = len(user_tasks)
                result["task_succeeded"] = len(set(result["task_succeeded"]).intersection(user_tasks))
                result["task_grades"] = {dg["taskid"]: dg["grade"] for dg in result["task_grades"] if dg["taskid"] in user_tasks}
                del result["_id"]
                return_data[username] = result
            return return_data
        else:
            return {}
Пример #28
0
    def page(self, course, error="", post=False):
        """ Get all data and display the page """
        groups = []
        if course.is_group_course():
            groups = get_database().groups.find({"course_id": course.get_id()})
        groups = sorted(groups, key=lambda item: item["description"])

        groups.insert(0, {"_id": 0, "users": course.get_staff(), "description": "Course staff", "tutors": {}})

        user_data = UserData.get_course_data_for_users(course.get_id(), course.get_registered_users())
        for user in user_data.keys():
            user_data[user]["url"] = self.submission_url_generator(course, user)

        users_csv = [dict(f.items() + [("username", username)]) for username, f in user_data.iteritems()]

        if "csv" in web.input():
            return make_csv(users_csv)

        return renderer.course_admin.student_list(course, user_data, groups, error, post)
Пример #29
0
 def page(self, course, username):
     """ Get all data and display the page """
     data = list(get_database().user_tasks.find({"username": username, "courseid": course.get_id()}))
     tasks = course.get_tasks()
     result = OrderedDict()
     for taskid in tasks:
         result[taskid] = {"name": tasks[taskid].get_name(), "submissions": 0, "status": "notviewed", "url": self.submission_url_generator(course, username, taskid)}
     for taskdata in data:
         if taskdata["taskid"] in result:
             result[taskdata["taskid"]]["submissions"] = taskdata["tried"]
             if taskdata["tried"] == 0:
                 result[taskdata["taskid"]]["status"] = "notattempted"
             elif taskdata["succeeded"]:
                 result[taskdata["taskid"]]["status"] = "succeeded"
             else:
                 result[taskdata["taskid"]]["status"] = "failed"
             result[taskdata["taskid"]]["grade"] = taskdata["grade"]
     if "csv" in web.input():
         return make_csv(result)
     return renderer.course_admin.student(course, username, result)
Пример #30
0
def add_batch_job(course, container_name, inputdata, launcher_name=None, skip_permission=False):
    """
        Add a job in the queue and returns a batch job id.
        inputdata is a dict containing all the keys of get_batch_container_metadata(container_name)[2] BUT the keys "course" and "submission" IF their
        type is "file". (the content of the course and the submission will be automatically included by this function.)
        The values associated are file-like objects for "file" types and  strings for "text" types.
    """

    if not skip_permission:
        if not User.is_logged_in():
            raise Exception("A user must be logged in to submit an object")

        username = User.get_username()
        launcher_name = launcher_name or username

        if username not in course.get_admins():
            raise Exception("The user must be an administrator to start a batch job")

    if container_name not in INGIniousConfiguration.get("batch_containers", []):
        raise Exception("This batch container is not allowed to be started")

    container_args = get_job_manager().get_batch_container_metadata(container_name)[2]
    if container_args is None:
        raise Exception("This batch container is not available")

    # Download the course content and submissions and add them to the input
    if "course" in container_args and container_args["course"]["type"] == "file" and "course" not in inputdata:
        inputdata["course"] = _get_course_data(course)
    if "submissions" in container_args and container_args["submissions"]["type"] == "file" and "submissions" not in inputdata:
        inputdata["submissions"] = _get_submissions_data(course)

    obj = {"courseid": course.get_id(), 'container_name': container_name, "submitted_on": datetime.now()}

    batch_job_id = get_database().batch_jobs.insert(obj)

    launcher_name = launcher_name or "plugin"

    get_job_manager().new_batch_job(container_name, inputdata, lambda r: _batch_job_done_callback(batch_job_id, r),
                                    launcher_name="Frontend - {}".format(launcher_name))

    return batch_job_id