def drop_batch_job(batch_job_id):
""" Delete a **finished** batch job from the database """
job = get_database().batch_jobs.find_one({"_id": ObjectId(batch_job_id)})
if "result" not in job:
raise Exception("Batch job is still running, cannot delete it")
get_database().batch_jobs.remove({"_id": ObjectId(batch_job_id)})
if "file" in job["result"]:
get_gridfs().delete(job["result"]["file"])
def add_job(task, inputdata, debug=False):
""" Add a job in the queue and returns a submission id.
task is a Task instance and inputdata is the input as a dictionary
If debug is true, more debug data will be saved
"""
if not User.is_logged_in():
raise Exception("A user must be logged in to submit an object")
username = User.get_username()
course = FrontendCourse(task.get_course_id())
obj = {
"courseid": task.get_course_id(),
"taskid": task.get_id(),
"input": get_gridfs().put(
json.dumps(inputdata)),
"status": "waiting",
"submitted_on": datetime.now()}
if course.is_group_course() and username not in course.get_staff(True):
group = get_database().groups.find_one({"course_id": task.get_course_id(), "users": username})
obj.update({"username": group["users"]})
else:
obj.update({"username": [username]})
submissionid = get_database().submissions.insert(obj)
PluginManager.get_instance().call_hook("new_submission", submissionid=submissionid, submission=obj, inputdata=inputdata)
get_job_manager().new_job(task, inputdata, (lambda job: _job_done_callback(submissionid, task, job)), "Frontend - {}".format(username), debug)
return submissionid
def _job_done_callback(submissionid, task, job):
""" Callback called by JobManager when a job is done. Updates the submission in the database with the data returned after the completion of the job """
submission = get_submission(submissionid, False)
submission = get_input_from_submission(submission)
job = _parse_text(task, job)
data = {
"status": ("done" if job["result"] == "success" or job["result"] == "failed" else "error"), # error only if error was made by INGInious
"result": job["result"],
"grade": job["grade"],
"text": job.get("text", None),
"tests": job.get("tests", None),
"problems": (job["problems"] if "problems" in job else {}),
"archive": (get_gridfs().put(base64.b64decode(job["archive"])) if "archive" in job else None)
}
# Store additional data
dont_dump = ["task", "course", "input"]
for index in job:
if index not in data and index not in dont_dump:
data[index] = job[index]
# Save submission to database
get_database().submissions.update(
{"_id": submission["_id"]},
{"$set": data}
)
for username in submission["username"]:
UserData(username).update_stats(submission, job)
PluginManager.get_instance().call_hook("submission_done", submission=submission, job=job)
def add_job(task, inputdata, debug=False):
""" Add a job in the queue and returns a submission id.
task is a Task instance and inputdata is the input as a dictionary
If debug is true, more debug data will be saved
"""
if not User.is_logged_in():
raise Exception("A user must be logged in to submit an object")
username = User.get_username()
jobid = get_job_manager().new_job_id()
obj = {
"username": username,
"courseid": task.get_course_id(),
"taskid": task.get_id(),
"input": get_gridfs().put(
json.dumps(inputdata)),
"status": "waiting",
"jobid": jobid,
"submitted_on": datetime.now()}
submissionid = get_database().submissions.insert(obj)
PluginManager.get_instance().call_hook("new_submission", submissionid=submissionid, submission=obj, jobid=jobid, inputdata=inputdata)
get_job_manager().new_job(task, inputdata, job_done_callback, "Frontend - {}".format(username), jobid, debug)
return submissionid
def get_input_from_submission(submission, only_input=False):
""" Get the input of a submission. If only_input is False, returns the full submissions with a dictionnary object at the key "input". Else, returns only the dictionnary. """
if isinstance(submission.get("input", {}), dict):
if only_input:
return submission.get("input", {})
else:
return submission
else:
inp = json.load(get_gridfs().get(submission['input']))
if only_input:
return inp
else:
submission["input"] = inp
return submission
def GET(self, courseid, bid, path=""):
""" GET request """
course, _ = get_course_and_check_rights(courseid)
batch_job = get_batch_job_status(bid)
if batch_job is None:
raise web.notfound()
if "result" not in batch_job or "file" not in batch_job["result"]:
raise web.notfound()
f = get_gridfs().get(batch_job["result"]["file"])
#hack for index.html:
if path == "/":
path = "/index.html"
if path == "":
web.header('Content-Type', 'application/x-gzip', unique=True)
web.header('Content-Disposition', 'attachment; filename="' + bid + '.tar.gz"', unique=True)
return f.read()
else:
path = path[1:] #remove the first /
if path.endswith('/'): # remove the last / if it exists
path = path[0:-1]
try:
tar = tarfile.open(fileobj=f, mode='r:gz')
file_info = tar.getmember(path)
except:
raise web.notfound()
if file_info.isdir(): #tar.gz the dir and return it
tmp = tempfile.TemporaryFile()
new_tar = tarfile.open(fileobj=tmp,mode='w:gz')
for m in tar.getmembers():
new_tar.addfile(m, tar.extractfile(m))
new_tar.close()
tmp.seek(0)
return tmp
elif not file_info.isfile():
raise web.notfound()
else: #guess a mime type and send it to the browser
to_dl = tar.extractfile(path).read()
mimetypes.init()
mime_type = mimetypes.guess_type(urllib.pathname2url(path))
web.header('Content-Type', mime_type[0])
return to_dl
def GET(self, courseid, bid):
""" GET request """
course, _ = get_course_and_check_rights(courseid)
batch_job = get_batch_job_status(bid)
if batch_job is None:
raise web.notfound()
done = False
submitted_on = batch_job["submitted_on"]
container_name = batch_job["container_name"]
container_title = container_name
container_description = ""
file_list = None
retval = 0
stdout = ""
stderr = ""
try:
container_metadata = get_batch_container_metadata(container_name)
if container_metadata == (None, None, None):
container_title = container_metadata[0]
container_description = container_metadata[1]
except:
pass
if "result" in batch_job:
done = True
retval = batch_job["result"]["retval"]
stdout = batch_job["result"].get("stdout","")
stderr = batch_job["result"].get("stderr", "")
if "file" in batch_job["result"]:
f = get_gridfs().get(batch_job["result"]["file"])
try:
tar = tarfile.open(fileobj=f,mode='r:gz')
file_list = set(tar.getnames()) - set([''])
tar.close()
except:
pass
finally:
f.close()
return renderer.course_admin.batch_summary(course, bid, done, container_name, container_title, container_description, submitted_on,
retval, stdout, stderr, file_list)
def _batch_job_done_callback(batch_job_id, result):
""" Called when the batch job with id jobid has finished.
result is a dictionnary, containing:
- {"retval": 0, "stdout": "...", "stderr": "...", "file": "..."}
if everything went well.(where file is a tgz file containing the content of the / output folder from the container)
- {"retval": "...", "stdout": "...", "stderr": "..."}
if the container crashed (retval is an int != 0)
- {"retval": -1, "stderr": "the error message"}
if the container failed to start
"""
# If there is a tgz file to save, put it in gridfs
if "file" in result:
result["file"] = get_gridfs().put(result["file"].read())
# Save submission to database
get_database().batch_jobs.update(
{"_id": batch_job_id},
{"$set": {"result": result}}
)
def get_submission_archive(submissions, sub_folders):
"""
:param submissions: a list of submissions
:param sub_folders: possible values:
[]: put all submissions in /
['taskid']: put all submissions for each task in a different directory /taskid/
['username']: put all submissions for each user in a different directory /username/
['taskid','username']: /taskid/username/
['username','taskid']: /username/taskid/
:return: a file-like object containing a tgz archive of all the submissions
"""
tmpfile = tempfile.TemporaryFile()
tar = tarfile.open(fileobj=tmpfile, mode='w:gz')
for submission in submissions:
submission = get_input_from_submission(submission)
# Compute base path in the tar file
base_path = "/"
for sub_folder in sub_folders:
if sub_folder == 'taskid':
base_path = submission['taskid'] + '/' + base_path
elif sub_folder == 'username':
base_path = submission['username'] + '/' + base_path
submission_yaml = StringIO.StringIO(common.custom_yaml.dump(submission).encode('utf-8'))
submission_yaml_fname = base_path + str(submission["_id"]) + '.test'
info = tarfile.TarInfo(name=submission_yaml_fname)
info.size = submission_yaml.len
info.mtime = time.mktime(submission["submitted_on"].timetuple())
# Add file in tar archive
tar.addfile(info, fileobj=submission_yaml)
# If there is an archive, add it too
if 'archive' in submission and submission['archive'] is not None and submission['archive'] != "":
subfile = get_gridfs().get(submission['archive'])
taskfname = base_path + str(submission["_id"]) + '.tgz'
# Generate file info
info = tarfile.TarInfo(name=taskfname)
info.size = subfile.length
info.mtime = time.mktime(submission["submitted_on"].timetuple())
# Add file in tar archive
tar.addfile(info, fileobj=subfile)
# If there files that were uploaded by the student, add them
if submission['input'] is not None:
for pid, problem in submission['input'].iteritems():
# If problem is a dict, it is a file (from the specification of the problems)
if isinstance(problem, dict):
# Get the extension (match extensions with more than one dot too)
DOUBLE_EXTENSIONS = ['.tar.gz', '.tar.bz2', '.tar.bz', '.tar.xz']
if not problem['filename'].endswith(tuple(DOUBLE_EXTENSIONS)):
_, ext = os.path.splitext(problem['filename'])
else:
for t_ext in DOUBLE_EXTENSIONS:
if problem['filename'].endswith(t_ext):
ext = t_ext
subfile = StringIO.StringIO(base64.b64decode(problem['value']))
taskfname = base_path + str(submission["_id"]) + '_uploaded_files/' + pid + ext
# Generate file info
info = tarfile.TarInfo(name=taskfname)
info.size = subfile.len
info.mtime = time.mktime(submission["submitted_on"].timetuple())
# Add file in tar archive
tar.addfile(info, fileobj=subfile)
# Close tarfile and put tempfile cursor at 0
tar.close()
tmpfile.seek(0)
return tmpfile
def download_submission_set(self, submissions, filename, sub_folders):
""" Create a tar archive with all the submissions """
if len(submissions) == 0:
raise web.notfound(renderer.notfound("There's no submission that matches your request"))
try:
tmpfile = tempfile.TemporaryFile()
tar = tarfile.open(fileobj=tmpfile, mode='w:gz')
for submission in submissions:
submission = get_input_from_submission(submission)
# Compute base path in the tar file
base_path = "/"
for sub_folder in sub_folders:
if sub_folder == 'taskid':
base_path = submission['taskid'] + '/' + base_path
elif sub_folder == 'username':
base_path = submission['username'] + '/' + base_path
submission_yaml = StringIO.StringIO(common.custom_yaml.dump(submission).encode('utf-8'))
submission_yaml_fname = base_path + str(submission["_id"]) + '.test'
info = tarfile.TarInfo(name=submission_yaml_fname)
info.size = submission_yaml.len
info.mtime = time.mktime(submission["submitted_on"].timetuple())
# Add file in tar archive
tar.addfile(info, fileobj=submission_yaml)
# If there is an archive, add it too
if 'archive' in submission and submission['archive'] is not None and submission['archive'] != "":
subfile = get_gridfs().get(submission['archive'])
taskfname = base_path + str(submission["_id"]) + '.tgz'
# Generate file info
info = tarfile.TarInfo(name=taskfname)
info.size = subfile.length
info.mtime = time.mktime(submission["submitted_on"].timetuple())
# Add file in tar archive
tar.addfile(info, fileobj=subfile)
# If there files that were uploaded by the student, add them
if submission['input'] is not None:
for pid, problem in submission['input'].iteritems():
# If problem is a dict, it is a file (from the specification of the problems)
if isinstance(problem, dict):
# Get the extension (match extensions with more than one dot too)
DOUBLE_EXTENSIONS = ['.tar.gz', '.tar.bz2', '.tar.bz', '.tar.xz']
if not problem['filename'].endswith(tuple(DOUBLE_EXTENSIONS)):
_, ext = os.path.splitext(problem['filename'])
else:
for t_ext in DOUBLE_EXTENSIONS:
if problem['filename'].endswith(t_ext):
ext = t_ext
subfile = StringIO.StringIO(base64.b64decode(problem['value']))
taskfname = base_path + str(submission["_id"]) + '_uploaded_files/' + pid + ext
# Generate file info
info = tarfile.TarInfo(name=taskfname)
info.size = subfile.len
info.mtime = time.mktime(submission["submitted_on"].timetuple())
# Add file in tar archive
tar.addfile(info, fileobj=subfile)
# Close tarfile and put tempfile cursor at 0
tar.close()
tmpfile.seek(0)
web.header('Content-Type', 'application/x-gzip', unique=True)
web.header('Content-Disposition', 'attachment; filename="' + filename + '"', unique=True)
return tmpfile
except Exception as e:
print e
raise web.notfound()
