def createTestMachine(self, test_host):
'''
Create test machine entry. The configuration information
is expected to be some JSON dictionary, since this is
normally directly rendered in the machine details view.
'''
self.machine = TestMachine(last_contact=datetime.datetime.now(),
host=test_host,
config=json.dumps(
{'Operating system': 'Plan 9'}))
self.machine.save()
return self.machine
def create_test_machine(test_host):
'''
Create test machine entry. The configuration information
is expected to be some JSON dictionary, since this is
normally directly rendered in the machine details view.
'''
machine = TestMachine(last_contact=datetime.datetime.now(),
host=test_host,
config=json.dumps([
['Operating system', uccrap + 'Plan 9'],
]))
machine.save()
return machine
def testAssignmentSpecificTestMachine(self):
# Register two test machines T1 and T2
real_machine = self._registerExecutor()
fake_machine = TestMachine(host="127.0.0.2")
fake_machine.save()
# Assign each of them to a different assignment
self.openAssignment.test_machines.add(real_machine)
self.validatedAssignment.test_machines.add(fake_machine)
# Produce submission for the assignment linked to fake_machine
sub1 = Submission(assignment=self.validatedAssignment,
submitter=self.current_user.user,
state=Submission.TEST_COMPILE_PENDING,
file_upload=self.createSubmissionFile())
sub1.save()
# Run real_machine executor, should not react on this submission
old_sub1_state = sub1.state
self.assertEquals(False, self._runExecutor())
# Make sure that submission object was not touched, whatever the executor says
sub1 = Submission.objects.get(pk=sub1.pk)
self.assertEquals(old_sub1_state, sub1.state)
def createTestMachine(self, test_host):
'''
Create test machine entry. The configuration information
is expected to be some JSON dictionary, since this is
normally directly rendered in the machine details view.
'''
self.machine = TestMachine(
last_contact=datetime.datetime.now(),
host = test_host,
config=json.dumps({'Operating system':'Plan 9'}))
self.machine.save()
return self.machine
def testAssignmentSpecificTestMachine(self):
# Register two test machines T1 and T2
real_machine = self._registerExecutor()
fake_machine = TestMachine(host="127.0.0.2")
fake_machine.save()
# Assign each of them to a different assignment
self.openAssignment.test_machines.add(real_machine)
self.validatedAssignment.test_machines.add(fake_machine)
# Produce submission for the assignment linked to fake_machine
sub1 = Submission(
assignment=self.validatedAssignment,
submitter=self.current_user.user,
state=Submission.TEST_COMPILE_PENDING,
file_upload=self.createSubmissionFile()
)
sub1.save()
# Run real_machine executor, should not react on this submission
old_sub1_state = sub1.state
self.assertEquals(False, self._runExecutor())
# Make sure that submission object was not touched, whatever the executor says
sub1 = Submission.objects.get(pk=sub1.pk)
self.assertEquals(old_sub1_state, sub1.state)
def machines(request):
''' This is the view used by the executor.py scripts for sending machine details.
A visible shared secret in the request is no problem, since the executors come
from trusted networks. The secret only protects this view from outside foreigners.
POST requests are expected to contain the following parameters:
'Config',
'Secret',
'UUID'
'''
if request.method == "POST":
try:
secret = request.POST['Secret']
uuid = request.POST['UUID']
address = request.POST['Address']
except Exception as e:
logger.error(
"Error finding the neccessary data in the executor request: " +
str(e))
raise PermissionDenied
if secret != settings.JOB_EXECUTOR_SECRET:
raise PermissionDenied
try:
# Find machine database entry for this host
machine = TestMachine.objects.get(host=uuid)
machine.last_contact = datetime.now()
machine.save()
except:
# Machine is not known so far, create new record
machine = TestMachine(host=uuid,
address=address,
last_contact=datetime.now())
machine.save()
# POST request contains all relevant machine information
machine.config = request.POST['Config']
machine.save()
return HttpResponse(status=201)
else:
return HttpResponse(status=500)
def machines(request, secret):
''' This is the view used by the executor.py scripts for putting machine details.
A visible shared secret in the request is no problem, since the executors come
from trusted networks. The secret only protects this view from outside foreigners.
'''
if secret != JOB_EXECUTOR_SECRET:
raise PermissionDenied
if request.method == "POST":
try:
# Find machine database entry for this host
machine = TestMachine.objects.get(host=request.POST['Name'])
machine.last_contact = datetime.now()
machine.save()
except:
# Machine is not known so far, create new record
machine = TestMachine(host=request.POST['Name'],
last_contact=datetime.now())
machine.save()
# POST request contains all relevant machine information
machine.config = request.POST['Config']
machine.save()
return HttpResponse(status=201)
else:
return HttpResponse(status=500)
def machines(request, secret):
''' This is the view used by the executor.py scripts for putting machine details.
A visible shared secret in the request is no problem, since the executors come
from trusted networks. The secret only protects this view from outside foreigners.
'''
if secret != JOB_EXECUTOR_SECRET:
raise PermissionDenied
if request.method == "POST":
try:
# Find machine database entry for this host
machine = TestMachine.objects.get(host=request.POST['Name'])
machine.last_contact = datetime.now()
machine.save()
except:
# Machine is not known so far, create new record
machine = TestMachine(host=request.POST['Name'], last_contact=datetime.now())
machine.save()
# POST request contains all relevant machine information
machine.config = request.POST['Config']
machine.save()
return HttpResponse(status=201)
else:
return HttpResponse(status=500)
def machines(request):
''' This is the view used by the executor.py scripts for putting machine details.
A visible shared secret in the request is no problem, since the executors come
from trusted networks. The secret only protects this view from outside foreigners.
POST requests are expected to contain the following parameters:
'Config',
'Secret',
'UUID'
'''
if request.method == "POST":
print request.POST
try:
secret = request.POST['Secret']
uuid = request.POST['UUID']
address = request.POST['Address']
except Exception as e:
logger.error("Error finding the neccessary data in the executor request: "+str(e))
raise PermissionDenied
if secret != settings.JOB_EXECUTOR_SECRET:
raise PermissionDenied
try:
# Find machine database entry for this host
machine = TestMachine.objects.get(host=uuid)
machine.last_contact = datetime.now()
machine.save()
except:
# Machine is not known so far, create new record
machine = TestMachine(host=uuid, address=address, last_contact=datetime.now())
machine.save()
# POST request contains all relevant machine information
machine.config = request.POST['Config']
machine.save()
return HttpResponse(status=201)
else:
return HttpResponse(status=500)
class SubmitTestCase(LiveServerTestCase):
'''
A test case base class with several resources being prepared:
Users:
- self.admin
- self.teacher
- self.another_teacher
- self.tutor
- self.enrolled_students
- self.not_enrolled_students
- self.current_user (the one currently logged-in)
No user is logged-in after setup.
Courses:
- self.course (by self.teacher)
- self.anotherCourse (by self.another_teacher)
- self.inactiveCourse
- self.all_courses
Assignments:
- self.openAssignment (in self.course)
- self.validatedAssignment (in self.course)
- self.softDeadlinePassedAssignment (in self.course)
- self.hardDeadlinePassedAssignment (in self.course)
- self.unpublishedAssignment (in self.course)
- self.allAssignments
Gradings:
- self.passGrade
- self.failGrade
- self.passFailGrading
The class offers some convinience functions:
- createTestMachine(self, test_host)
- createSubmissionFile(self)
- createTestedSubmissionFile(self, test_machine)
- createValidatableSubmission(self, user)
- createValidatedSubmission(self, user, test_host='127.0.0.1')
- createSubmission(self, user, assignment)
'''
current_user = None
def setUp(self):
super(SubmitTestCase, self).setUp()
self.logger = logging.getLogger('OpenSubmit')
self.loggerLevelOld = self.logger.level
self.logger.setLevel(logging.DEBUG)
self.setUpUsers()
self.setUpCourses()
self.setUpGradings()
self.setUpAssignments()
def tearDown(self):
self.logger.setLevel(self.loggerLevelOld)
def createUser(self, user_dict):
args = dict(user_dict)
args['password'] = make_password(args['password'])
user_obj = User(**args)
user_obj.save()
user_profile = UserProfile(user=user_obj)
user_profile.save()
user_dict['user'] = user_obj
user_dict['profile'] = user_profile
user_struct = AnonStruct(user_dict)
return user_struct
def loginUser(self, user_struct):
assert (self.c.login(username=user_struct.username,
password=user_struct.password))
uid = self.c.session['_auth_user_id']
user_struct.user = User.objects.get(pk=uid)
self.current_user = user_struct
def setUpUsers(self):
self.c = Client()
self.admin_dict = {
'username': '******',
'password': '******',
'email': '*****@*****.**',
'is_staff': True,
'is_superuser': True
}
self.admin = self.createUser(self.admin_dict)
self.teacher_dict = {
'username': '******',
'password': '******',
'email': '*****@*****.**',
'is_staff': True,
'is_superuser': False
}
self.teacher = self.createUser(self.teacher_dict)
self.another_teacher_dict = {
'username': '******',
'password': '******',
'email': '*****@*****.**',
'is_staff': True,
'is_superuser': False
}
self.another_teacher = self.createUser(self.another_teacher_dict)
self.tutor_dict = {
'username': '******',
'password': '******',
'email': '*****@*****.**',
'is_staff': True,
'is_superuser': False
}
self.tutor = self.createUser(self.tutor_dict)
self.enrolled_students = list()
for i in range(0, 5):
enrolled_student_dict = {
'username':
'******'.format(i),
'password':
'******'.format(i),
'email':
'testrunner_enrolled_student{}@django.localhost.local'.format(
i),
'is_staff':
False,
'is_superuser':
False,
'first_name':
'Harold',
'last_name':
'Finch'
}
self.enrolled_students.append(
self.createUser(enrolled_student_dict))
self.not_enrolled_students = list()
for i in range(0, 5):
not_enrolled_student_dict = {
'username':
'******'.format(i),
'password':
'******'.format(i),
'email':
'testrunner_not_enrolled_student{}@django.localhost.local'.
format(i),
'is_staff':
False,
'is_superuser':
False
}
self.not_enrolled_students.append(
self.createUser(not_enrolled_student_dict))
def setUpCourses(self):
self.all_courses = []
self.course = Course(
title='Active test course',
active=True,
owner=self.teacher.user,
max_authors=3,
)
self.course.save()
self.course.tutors.add(self.tutor.user)
for student in self.enrolled_students:
self.course.participants.add(student.profile)
self.all_courses.append(self.course)
self.anotherCourse = Course(
title='Another active test course',
active=True,
owner=self.another_teacher.user,
max_authors=1,
)
self.anotherCourse.save()
self.all_courses.append(self.anotherCourse)
self.inactiveCourse = Course(
title='Inactive test course',
active=False,
owner=self.another_teacher.user,
max_authors=1,
)
self.inactiveCourse.save()
self.all_courses.append(self.inactiveCourse)
def setUpGradings(self):
self.passGrade = Grading(title='passed', means_passed=True)
self.passGrade.save()
self.failGrade = Grading(title='failed', means_passed=False)
self.failGrade.save()
self.passFailGrading = GradingScheme(title='Pass/Fail Grading Scheme')
self.passFailGrading.save()
self.passFailGrading.gradings.add(self.passGrade)
self.passFailGrading.gradings.add(self.failGrade)
self.passFailGrading.save()
def setUpAssignments(self):
from django.core.files import File as DjangoFile
today = timezone.now()
last_week = today - datetime.timedelta(weeks=1)
yesterday = today - datetime.timedelta(days=1)
tomorrow = today + datetime.timedelta(days=1)
next_week = today + datetime.timedelta(weeks=1)
# List of all assignments being assigned to the "self.course" course
self.allAssignments = []
self.openAssignment = Assignment(
title='Open assignment',
course=self.course,
download='http://example.org/assignments/1/download',
gradingScheme=self.passFailGrading,
publish_at=last_week,
soft_deadline=tomorrow,
hard_deadline=next_week,
has_attachment=False)
self.openAssignment.save()
self.allAssignments.append(self.openAssignment)
self.anotherAssignment = Assignment(
title='Another open assignment',
course=self.anotherCourse,
download='http://example.org/assignments/1/download',
gradingScheme=self.passFailGrading,
publish_at=last_week,
soft_deadline=tomorrow,
hard_deadline=next_week,
has_attachment=False)
self.anotherAssignment.save()
self.fileAssignment = Assignment(
title='File assignment',
course=self.course,
download='http://example.org/assignments/1/download',
gradingScheme=self.passFailGrading,
publish_at=last_week,
soft_deadline=tomorrow,
hard_deadline=next_week,
has_attachment=True)
self.fileAssignment.save()
self.allAssignments.append(self.fileAssignment)
self.validatedAssignment = Assignment(
title='Validated assignment',
course=self.course,
download='http://example.org/assignments/1/download',
gradingScheme=self.passFailGrading,
publish_at=last_week,
soft_deadline=tomorrow,
hard_deadline=next_week,
has_attachment=True,
validity_script_download=True,
attachment_test_validity=DjangoFile(
open(rootdir + '/opensubmit/tests/validators/working.zip')),
attachment_test_full=DjangoFile(
open(rootdir + '/opensubmit/tests/validators/working.zip')))
self.validatedAssignment.save()
self.allAssignments.append(self.validatedAssignment)
self.softDeadlinePassedAssignment = Assignment(
title='Soft deadline passed assignment',
course=self.course,
download='http://example.org/assignments/2/download',
gradingScheme=self.passFailGrading,
publish_at=last_week,
soft_deadline=yesterday,
hard_deadline=tomorrow,
has_attachment=False,
)
self.softDeadlinePassedAssignment.save()
self.allAssignments.append(self.softDeadlinePassedAssignment)
self.hardDeadlinePassedAssignment = Assignment(
title='Hard deadline passed assignment',
course=self.course,
download='http://example.org/assignments/3/download',
gradingScheme=self.passFailGrading,
publish_at=last_week,
soft_deadline=yesterday,
hard_deadline=yesterday,
has_attachment=False,
)
self.hardDeadlinePassedAssignment.save()
self.allAssignments.append(self.hardDeadlinePassedAssignment)
self.unpublishedAssignment = Assignment(
title='Unpublished assignment',
course=self.course,
download='http://example.org/assignments/4/download',
gradingScheme=self.passFailGrading,
publish_at=tomorrow,
soft_deadline=next_week,
hard_deadline=next_week,
has_attachment=False,
)
self.unpublishedAssignment.save()
self.allAssignments.append(self.unpublishedAssignment)
def createTestMachine(self, test_host):
'''
Create test machine entry. The configuration information
is expected to be some JSON dictionary, since this is
normally directly rendered in the machine details view.
'''
self.machine = TestMachine(last_contact=datetime.datetime.now(),
host=test_host,
config=json.dumps(
{'Operating system': 'Plan 9'}))
self.machine.save()
return self.machine
def createSubmissionFile(
self,
relpath="/opensubmit/tests/submfiles/working_withsubdir.zip"):
from django.core.files import File as DjangoFile
fname = relpath[relpath.rfind(os.sep) + 1:]
shutil.copyfile(rootdir + relpath, settings.MEDIA_ROOT + fname)
sf = SubmissionFile(
attachment=DjangoFile(open(rootdir + relpath), unicode(fname)))
sf.save()
return sf
def createTestedSubmissionFile(self, test_machine):
'''
Create finalized test result in the database.
'''
sf = self.createSubmissionFile()
result_compile = SubmissionTestResult(
kind=SubmissionTestResult.COMPILE_TEST,
result="Compilation ok.",
machine=test_machine,
submission_file=sf).save()
result_validity = SubmissionTestResult(
kind=SubmissionTestResult.VALIDITY_TEST,
result="Validation ok.",
machine=self.machine,
perf_data="41;42;43",
submission_file=sf).save()
result_full = SubmissionTestResult(kind=SubmissionTestResult.FULL_TEST,
result="Full test ok.",
perf_data="77;88;99",
machine=self.machine,
submission_file=sf).save()
return sf
def createValidatableSubmission(self, user):
'''
Create a submission that can be validated by executor.
'''
sf = self.createSubmissionFile()
sub = Submission(assignment=self.validatedAssignment,
submitter=user.user,
notes="This is a validatable submission.",
state=Submission.TEST_COMPILE_PENDING,
file_upload=sf)
sub.save()
return sub
def createValidatedSubmission(self, user, test_host='127.0.0.1'):
'''
Create a submission that already has test results in the database.
'''
machine = self.createTestMachine(test_host)
sf = self.createTestedSubmissionFile(machine)
sub = Submission(assignment=self.validatedAssignment,
submitter=user.user,
notes="This is an already validated submission.",
state=Submission.SUBMITTED_TESTED,
file_upload=sf)
sub.save()
return sub
def createSubmission(self, user, assignment):
sub = Submission(assignment=assignment,
submitter=user.user,
notes="This is a submission.",
state=Submission.SUBMITTED)
sub.save()
return sub
def jobs(request, secret):
''' This is the view used by the executor.py scripts for getting / putting the test results.
Fetching some file for testing is changing the database, so using GET here is not really RESTish. Whatever.
A visible shared secret in the request is no problem, since the executors come
from trusted networks. The secret only protects this view from outside foreigners.
'''
if secret != JOB_EXECUTOR_SECRET:
raise PermissionDenied
if request.method == "GET":
try:
machine = TestMachine.objects.get(host=request.get_host())
machine.last_contact = datetime.now()
machine.save()
except:
# ask for configuration of new execution hosts by returning the according action
machine = TestMachine(host=request.get_host(), last_contact=datetime.now())
machine.save()
response = HttpResponse()
response['Action'] = 'get_config'
response['MachineId'] = machine.pk
return response
subm = Submission.pending_student_tests.all()
if len(subm) == 0:
subm = Submission.pending_full_tests.all()
if len(subm) == 0:
raise Http404
for sub in subm:
assert (sub.file_upload) # must be given when the state model is correct
# only deliver jobs that are unfetched so far, or where the executor should have finished meanwhile
# it may happen in special cases that stucked executors deliver their result after the timeout
# this is not really a problem, since the result remains the same for the same file
# TODO: Make this a part of the original query
# TODO: Count number of attempts to leave the same state, mark as finally failed in case; alternatively, the executor must always deliver a re.
if (not sub.file_upload.fetched) or (sub.file_upload.fetched + datetime.timedelta(
seconds=sub.assignment.attachment_test_timeout) < timezone.now()):
if sub.file_upload.fetched:
# Stuff that has timed out
# we mark it as failed so that the user gets informed
# TODO: Late delivery for such a submission by the executor witll break everything
sub.file_upload.fetched = None
if sub.state == Submission.TEST_COMPILE_PENDING:
sub.state = Submission.TEST_COMPILE_FAILED
sub.file_upload.test_compile = "Killed due to non-reaction on timeout signals. Please check your application for deadlocks or keyboard input."
inform_student(sub, sub.state)
if sub.state == Submission.TEST_VALIDITY_PENDING:
sub.file_upload.test_validity = "Killed due to non-reaction on timeout signals. Please check your application for deadlocks or keyboard input."
sub.state = Submission.TEST_VALIDITY_FAILED
inform_student(sub, sub.state)
if sub.state == Submission.TEST_FULL_PENDING:
sub.file_upload.test_full = "Killed due to non-reaction on timeout signals. Student not informed, since this was the full test."
sub.state = Submission.TEST_FULL_FAILED
sub.file_upload.save()
sub.save()
continue
# create HTTP response with file download
f = sub.file_upload.attachment
# on dev server, we sometimes have stale database entries
if not os.access(f.path, os.F_OK):
mail_managers('Warning: Missing file',
'Missing file on storage for submission file entry %u: %s' % (
sub.file_upload.pk, str(sub.file_upload.attachment)), fail_silently=True)
continue
response = HttpResponse(f, content_type='application/binary')
response['Content-Disposition'] = 'attachment; filename="%s"' % sub.file_upload.basename()
response['SubmissionFileId'] = str(sub.file_upload.pk)
response['Timeout'] = sub.assignment.attachment_test_timeout
if sub.state == Submission.TEST_COMPILE_PENDING:
response['Action'] = 'test_compile'
elif sub.state == Submission.TEST_VALIDITY_PENDING:
response['Action'] = 'test_validity'
# reverse() is messing up here when we have to FORCE_SCRIPT case, so we do manual URL construction
response['PostRunValidation'] = MAIN_URL + "/download/%u/validity_testscript/secret=%s" % (
sub.assignment.pk, JOB_EXECUTOR_SECRET)
elif sub.state == Submission.TEST_FULL_PENDING or sub.state == Submission.CLOSED_TEST_FULL_PENDING:
response['Action'] = 'test_full'
# reverse() is messing up here when we have to FORCE_SCRIPT case, so we do manual URL construction
response['PostRunValidation'] = MAIN_URL + "/download/%u/full_testscript/secret=%s" % (
sub.assignment.pk, JOB_EXECUTOR_SECRET)
else:
assert (False)
# store date of fetching for determining jobs stucked at the executor
sub.file_upload.fetched = timezone.now()
sub.file_upload.save()
# 'touch' submission so that it becomes sorted to the end of the queue if something goes wrong
sub.modified = timezone.now()
sub.save()
return response
# no feasible match in the list of possible jobs
raise Http404
elif request.method == "POST":
# first check if this is just configuration data, and not a job result
if request.POST['Action'] == 'get_config':
machine = TestMachine.objects.get(pk=int(request.POST['MachineId']))
machine.config = request.POST['Config']
machine.save()
return HttpResponse(status=201)
# executor.py is providing the results as POST parameters
sid = request.POST['SubmissionFileId']
submission_file = get_object_or_404(SubmissionFile, pk=sid)
sub = submission_file.submissions.all()[0]
error_code = int(request.POST['ErrorCode'])
if request.POST['Action'] == 'test_compile' and sub.state == Submission.TEST_COMPILE_PENDING:
submission_file.test_compile = request.POST['Message']
if error_code == 0:
if sub.assignment.attachment_test_validity:
sub.state = Submission.TEST_VALIDITY_PENDING
elif sub.assignment.attachment_test_full:
sub.state = Submission.TEST_FULL_PENDING
else:
sub.state = Submission.SUBMITTED_TESTED
inform_course_owner(request, sub)
else:
sub.state = Submission.TEST_COMPILE_FAILED
inform_student(sub, sub.state)
elif request.POST['Action'] == 'test_validity' and sub.state == Submission.TEST_VALIDITY_PENDING:
submission_file.test_validity = request.POST['Message']
if error_code == 0:
if sub.assignment.attachment_test_full:
sub.state = Submission.TEST_FULL_PENDING
else:
sub.state = Submission.SUBMITTED_TESTED
inform_course_owner(request, sub)
else:
sub.state = Submission.TEST_VALIDITY_FAILED
inform_student(sub, sub.state)
elif request.POST['Action'] == 'test_full' and sub.state == Submission.TEST_FULL_PENDING:
submission_file.test_full = request.POST['Message']
if error_code == 0:
sub.state = Submission.SUBMITTED_TESTED
inform_course_owner(request, sub)
else:
sub.state = Submission.TEST_FULL_FAILED
# full tests may be performed several times and are meant to be a silent activity
# therefore, we send no mail to the student here
elif request.POST['Action'] == 'test_full' and sub.state == Submission.CLOSED_TEST_FULL_PENDING:
submission_file.test_full = request.POST['Message']
sub.state = Submission.CLOSED
# full tests may be performed several times and are meant to be a silent activity
# therefore, we send no mail to the student here
else:
mail_managers('Warning: Inconsistent job state', str(sub.pk), fail_silently=True)
submission_file.fetched = None # makes the file fetchable again by executors, but now in a different state
perf_data = request.POST['PerfData'].strip()
if perf_data != "":
submission_file.perf_data = perf_data
else:
submission_file.perf_data = None
submission_file.save()
sub.save()
return HttpResponse(status=201)
import datetime
import json
from django.core.management.base import BaseCommand
from django.contrib.auth.models import User
from opensubmit.models import Course, Assignment, Grading, GradingScheme, Submission, SubmissionFile, SubmissionTestResult, TestMachine
from opensubmit import settings
from django.utils import timezone
from django.core.files import File as DjangoFile
from tempfile import NamedTemporaryFile
# test machine
machine = TestMachine(
last_contact=datetime.datetime.now(),
host='UUID4711',
config=json.dumps([["Operating system", "Plan 9"], ]))
machine.save()
def createSubmissionFile():
with NamedTemporaryFile(mode="wt", delete=False, prefix=settings.MEDIA_ROOT) as tmpfile:
# Submission file
tmpfile.write("The quick brown fox jumps over the lazy dog.")
tmpfile.close()
sf = SubmissionFile(attachment=DjangoFile(tmpfile.name))
sf.save()
# os.remove(tmpfile.name)
# Test results
val_result = SubmissionTestResult()
val_result.submission_file = sf
val_result.kind = SubmissionTestResult.VALIDITY_TEST
val_result.result = "Validation test result for student"
def jobs(request, secret):
''' This is the view used by the executor.py scripts for getting / putting the test results.
Fetching some file for testing is changing the database, so using GET here is not really RESTish. Whatever.
A visible shared secret in the request is no problem, since the executors come
from trusted networks. The secret only protects this view from outside foreigners.
'''
if secret != JOB_EXECUTOR_SECRET:
raise PermissionDenied
if request.method == "GET":
try:
machine = TestMachine.objects.get(host=request.get_host())
machine.last_contact = datetime.now()
machine.save()
except:
# ask for configuration of new execution hosts by returning the according action
machine = TestMachine(host=request.get_host(),
last_contact=datetime.now())
machine.save()
response = HttpResponse()
response['Action'] = 'get_config'
response['MachineId'] = machine.pk
return response
subm = Submission.pending_student_tests.all()
if len(subm) == 0:
subm = Submission.pending_full_tests.all()
if len(subm) == 0:
raise Http404
for sub in subm:
assert (sub.file_upload
) # must be given when the state model is correct
# only deliver jobs that are unfetched so far, or where the executor should have finished meanwhile
# it may happen in special cases that stucked executors deliver their result after the timeout
# this is not really a problem, since the result remains the same for the same file
# TODO: Make this a part of the original query
# TODO: Count number of attempts to leave the same state, mark as finally failed in case; alternatively, the executor must always deliver a re.
if (not sub.file_upload.fetched) or (
sub.file_upload.fetched + datetime.timedelta(
seconds=sub.assignment.attachment_test_timeout) <
timezone.now()):
if sub.file_upload.fetched:
# Stuff that has timed out
# we mark it as failed so that the user gets informed
# TODO: Late delivery for such a submission by the executor witll break everything
sub.file_upload.fetched = None
if sub.state == Submission.TEST_COMPILE_PENDING:
sub.state = Submission.TEST_COMPILE_FAILED
sub.file_upload.test_compile = "Killed due to non-reaction on timeout signals. Please check your application for deadlocks or keyboard input."
inform_student(sub, sub.state)
if sub.state == Submission.TEST_VALIDITY_PENDING:
sub.file_upload.test_validity = "Killed due to non-reaction on timeout signals. Please check your application for deadlocks or keyboard input."
sub.state = Submission.TEST_VALIDITY_FAILED
inform_student(sub, sub.state)
if sub.state == Submission.TEST_FULL_PENDING:
sub.file_upload.test_full = "Killed due to non-reaction on timeout signals. Student not informed, since this was the full test."
sub.state = Submission.TEST_FULL_FAILED
sub.file_upload.save()
sub.save()
continue
# create HTTP response with file download
f = sub.file_upload.attachment
# on dev server, we sometimes have stale database entries
if not os.access(f.path, os.F_OK):
mail_managers(
'Warning: Missing file',
'Missing file on storage for submission file entry %u: %s'
%
(sub.file_upload.pk, str(sub.file_upload.attachment)),
fail_silently=True)
continue
response = HttpResponse(f, content_type='application/binary')
response[
'Content-Disposition'] = 'attachment; filename="%s"' % sub.file_upload.basename(
)
response['SubmissionFileId'] = str(sub.file_upload.pk)
response['Timeout'] = sub.assignment.attachment_test_timeout
if sub.state == Submission.TEST_COMPILE_PENDING:
response['Action'] = 'test_compile'
elif sub.state == Submission.TEST_VALIDITY_PENDING:
response['Action'] = 'test_validity'
# reverse() is messing up here when we have to FORCE_SCRIPT case, so we do manual URL construction
response[
'PostRunValidation'] = MAIN_URL + "/download/%u/validity_testscript/secret=%s" % (
sub.assignment.pk, JOB_EXECUTOR_SECRET)
elif sub.state == Submission.TEST_FULL_PENDING or sub.state == Submission.CLOSED_TEST_FULL_PENDING:
response['Action'] = 'test_full'
# reverse() is messing up here when we have to FORCE_SCRIPT case, so we do manual URL construction
response[
'PostRunValidation'] = MAIN_URL + "/download/%u/full_testscript/secret=%s" % (
sub.assignment.pk, JOB_EXECUTOR_SECRET)
else:
assert (False)
# store date of fetching for determining jobs stucked at the executor
sub.file_upload.fetched = timezone.now()
sub.file_upload.save()
# 'touch' submission so that it becomes sorted to the end of the queue if something goes wrong
sub.modified = timezone.now()
sub.save()
return response
# no feasible match in the list of possible jobs
raise Http404
elif request.method == "POST":
# first check if this is just configuration data, and not a job result
if request.POST['Action'] == 'get_config':
machine = TestMachine.objects.get(
pk=int(request.POST['MachineId']))
machine.config = request.POST['Config']
machine.save()
return HttpResponse(status=201)
# executor.py is providing the results as POST parameters
sid = request.POST['SubmissionFileId']
submission_file = get_object_or_404(SubmissionFile, pk=sid)
sub = submission_file.submissions.all()[0]
error_code = int(request.POST['ErrorCode'])
if request.POST[
'Action'] == 'test_compile' and sub.state == Submission.TEST_COMPILE_PENDING:
submission_file.test_compile = request.POST['Message']
if error_code == 0:
if sub.assignment.attachment_test_validity:
sub.state = Submission.TEST_VALIDITY_PENDING
elif sub.assignment.attachment_test_full:
sub.state = Submission.TEST_FULL_PENDING
else:
sub.state = Submission.SUBMITTED_TESTED
inform_course_owner(request, sub)
else:
sub.state = Submission.TEST_COMPILE_FAILED
inform_student(sub, sub.state)
elif request.POST[
'Action'] == 'test_validity' and sub.state == Submission.TEST_VALIDITY_PENDING:
submission_file.test_validity = request.POST['Message']
if error_code == 0:
if sub.assignment.attachment_test_full:
sub.state = Submission.TEST_FULL_PENDING
else:
sub.state = Submission.SUBMITTED_TESTED
inform_course_owner(request, sub)
else:
sub.state = Submission.TEST_VALIDITY_FAILED
inform_student(sub, sub.state)
elif request.POST[
'Action'] == 'test_full' and sub.state == Submission.TEST_FULL_PENDING:
submission_file.test_full = request.POST['Message']
if error_code == 0:
sub.state = Submission.SUBMITTED_TESTED
inform_course_owner(request, sub)
else:
sub.state = Submission.TEST_FULL_FAILED
# full tests may be performed several times and are meant to be a silent activity
# therefore, we send no mail to the student here
elif request.POST[
'Action'] == 'test_full' and sub.state == Submission.CLOSED_TEST_FULL_PENDING:
submission_file.test_full = request.POST['Message']
sub.state = Submission.CLOSED
# full tests may be performed several times and are meant to be a silent activity
# therefore, we send no mail to the student here
else:
mail_managers('Warning: Inconsistent job state',
str(sub.pk),
fail_silently=True)
submission_file.fetched = None # makes the file fetchable again by executors, but now in a different state
perf_data = request.POST['PerfData'].strip()
if perf_data != "":
submission_file.perf_data = perf_data
else:
submission_file.perf_data = None
submission_file.save()
sub.save()
return HttpResponse(status=201)
class SubmitTestCase(TestCase):
current_user = None
def createUser(self, user_dict):
args = dict(user_dict)
args['password'] = make_password(args['password'])
user_obj = User(**args)
user_obj.save()
user_profile = UserProfile(user=user_obj)
user_profile.save()
user_dict['user'] = user_obj
user_dict['profile'] = user_profile
user_struct = AnonStruct(user_dict)
return user_struct
def loginUser(self, user_struct):
assert(self.c.login(username=user_struct.username, password=user_struct.password))
uid = self.c.session['_auth_user_id']
user_struct.user = User.objects.get(pk=uid)
self.current_user = user_struct
def setUpUsers(self):
self.c = Client()
self.admin_dict = {
'username': '******',
'password': '******',
'email': '*****@*****.**',
'is_staff': True,
'is_superuser': True
}
self.admin = self.createUser(self.admin_dict)
self.teacher_dict = {
'username': '******',
'password': '******',
'email': '*****@*****.**',
'is_staff': True,
'is_superuser': False
}
self.teacher = self.createUser(self.teacher_dict)
self.another_teacher_dict = {
'username': '******',
'password': '******',
'email': '*****@*****.**',
'is_staff': True,
'is_superuser': False
}
self.another_teacher = self.createUser(self.another_teacher_dict)
self.tutor_dict = {
'username': '******',
'password': '******',
'email': '*****@*****.**',
'is_staff': True,
'is_superuser': False
}
self.tutor = self.createUser(self.tutor_dict)
self.enrolled_students = list()
for i in range(0, 5):
enrolled_student_dict = {
'username': '******'.format(i),
'password': '******'.format(i),
'email': 'testrunner_enrolled_student{}@django.localhost.local'.format(i),
'is_staff': False,
'is_superuser': False,
'first_name': 'Harold',
'last_name': 'Finch'
}
self.enrolled_students.append(self.createUser(enrolled_student_dict))
self.not_enrolled_students = list()
for i in range(0, 5):
not_enrolled_student_dict = {
'username': '******'.format(i),
'password': '******'.format(i),
'email': 'testrunner_not_enrolled_student{}@django.localhost.local'.format(i),
'is_staff': False,
'is_superuser': False
}
self.not_enrolled_students.append(self.createUser(not_enrolled_student_dict))
def setUpCourses(self):
self.all_courses = []
self.course = Course(
title='Active test course',
active=True,
owner=self.teacher.user,
max_authors=3,
)
self.course.save()
self.course.tutors.add(self.tutor.user)
for student in self.enrolled_students:
self.course.participants.add(student.profile)
self.all_courses.append(self.course)
self.anotherCourse = Course(
title='Another active test course',
active=True,
owner=self.another_teacher.user,
max_authors=1,
)
self.anotherCourse.save()
self.all_courses.append(self.anotherCourse)
self.inactiveCourse = Course(
title='Inactive test course',
active=False,
owner=self.another_teacher.user,
max_authors=1,
)
self.inactiveCourse.save()
self.all_courses.append(self.inactiveCourse)
def setUpGradings(self):
self.passGrade = Grading(title='passed', means_passed=True)
self.passGrade.save()
self.failGrade = Grading(title='failed', means_passed=False)
self.failGrade.save()
self.passFailGrading = GradingScheme(title='Pass/Fail Grading Scheme')
self.passFailGrading.save()
self.passFailGrading.gradings.add(self.passGrade)
self.passFailGrading.gradings.add(self.failGrade)
self.passFailGrading.save()
def setUpAssignments(self):
from django.core.files import File as DjangoFile
today = timezone.now()
last_week = today - datetime.timedelta(weeks=1)
yesterday = today - datetime.timedelta(days=1)
tomorrow = today + datetime.timedelta(days=1)
next_week = today + datetime.timedelta(weeks=1)
self.allAssignments = []
self.openAssignment = Assignment(
title='Open assignment',
course=self.course,
download='http://example.org/assignments/1/download',
gradingScheme=self.passFailGrading,
publish_at=last_week,
soft_deadline=tomorrow,
hard_deadline=next_week,
has_attachment=False
)
self.openAssignment.save()
self.allAssignments.append(self.openAssignment)
self.validatedAssignment = Assignment(
title='Validated assignment',
course=self.course,
download='http://example.org/assignments/1/download',
gradingScheme=self.passFailGrading,
publish_at=last_week,
soft_deadline=tomorrow,
hard_deadline=next_week,
has_attachment=True,
validity_script_download=True,
attachment_test_validity=DjangoFile(open(rootdir+'/opensubmit/tests/validators/working.zip')),
attachment_test_full=DjangoFile(open(rootdir+'/opensubmit/tests/validators/working.zip'))
)
self.validatedAssignment.save()
self.allAssignments.append(self.validatedAssignment)
self.softDeadlinePassedAssignment = Assignment(
title='Soft deadline passed assignment',
course=self.course,
download='http://example.org/assignments/2/download',
gradingScheme=self.passFailGrading,
publish_at=last_week,
soft_deadline=yesterday,
hard_deadline=tomorrow,
has_attachment=False,
)
self.softDeadlinePassedAssignment.save()
self.allAssignments.append(self.softDeadlinePassedAssignment)
self.hardDeadlinePassedAssignment = Assignment(
title='Hard deadline passed assignment',
course=self.course,
download='http://example.org/assignments/3/download',
gradingScheme=self.passFailGrading,
publish_at=last_week,
soft_deadline=yesterday,
hard_deadline=yesterday,
has_attachment=False,
)
self.hardDeadlinePassedAssignment.save()
self.allAssignments.append(self.hardDeadlinePassedAssignment)
self.unpublishedAssignment = Assignment(
title='Unpublished assignment',
course=self.course,
download='http://example.org/assignments/4/download',
gradingScheme=self.passFailGrading,
publish_at=tomorrow,
soft_deadline=next_week,
hard_deadline=next_week,
has_attachment=False,
)
self.unpublishedAssignment.save()
self.allAssignments.append(self.unpublishedAssignment)
def setUp(self):
super(SubmitTestCase, self).setUp()
self.logger = logging.getLogger('OpenSubmit')
self.loggerLevelOld = self.logger.level
self.logger.setLevel(logging.WARN)
self.setUpUsers()
self.setUpCourses()
self.setUpGradings()
self.setUpAssignments()
def tearDown(self):
self.logger.setLevel(self.loggerLevelOld)
def createTestMachine(self, test_host):
'''
Create test machine entry. The configuration information
is expected to be some JSON dictionary, since this is
normally directly rendered in the machine details view.
'''
self.machine = TestMachine(
last_contact=datetime.datetime.now(),
host = test_host,
config=json.dumps({'Operating system':'Plan 9'}))
self.machine.save()
return self.machine
def createSubmissionFile(self):
from django.core.files import File as DjangoFile
sf = SubmissionFile(attachment=DjangoFile(open(rootdir+"/opensubmit/tests/submfiles/working_withsubdir.zip"), unicode("working_withsubdir.zip")))
sf.save()
return sf
def createTestedSubmissionFile(self, test_machine):
'''
Create finalized test result in the database.
'''
sf = self.createSubmissionFile()
result_compile = SubmissionTestResult(
kind=SubmissionTestResult.COMPILE_TEST,
result="Compilation ok.",
machine=test_machine,
submission_file=sf
).save()
result_validity = SubmissionTestResult(
kind=SubmissionTestResult.VALIDITY_TEST,
result="Validation ok.",
machine=self.machine,
perf_data = "41;42;43",
submission_file=sf).save()
result_full = SubmissionTestResult(
kind=SubmissionTestResult.FULL_TEST,
result="Full test ok.",
perf_data = "77;88;99",
machine=self.machine,
submission_file=sf).save()
return sf
def createValidatableSubmission(self, user):
'''
Create a submission that can be validated by executor.
'''
sf = self.createSubmissionFile()
sub = Submission(
assignment=self.validatedAssignment,
submitter=user.user,
notes="This is a validatable submission.",
state=Submission.TEST_COMPILE_PENDING,
file_upload=sf
)
sub.save()
return sub
def createValidatedSubmission(self, user, test_host='127.0.0.1'):
'''
Create a submission that already has test results in the database.
'''
machine = self.createTestMachine(test_host)
sf = self.createTestedSubmissionFile(machine)
sub = Submission(
assignment=self.validatedAssignment,
submitter=user.user,
notes="This is an already validated submission.",
state=Submission.SUBMITTED_TESTED,
file_upload=sf
)
sub.save()
return sub
def createSubmission(self, user, assignment, authors=[]):
sub = Submission(
assignment=assignment,
submitter=user.user,
notes="This is a submission.",
state=Submission.SUBMITTED
)
sub.save()
if authors:
[sub.authors.add(author) for author in authors]
sub.save()
return sub
def jobs(request):
''' This is the view used by the executor.py scripts for getting / putting the test results.
Fetching some file for testing is changing the database, so using GET here is not really RESTish. Whatever.
A visible shared secret in the request is no problem, since the executors come
from trusted networks. The secret only protects this view from outside foreigners.
TODO: Make it a real API, based on some framework.
TODO: Factor out state model from this method into some model.
POST requests with 'Action'='get_config' are expected to contain the following parameters:
'MachineId',
'Config',
'Secret',
'UUID'
All other POST requests are expected to contain the following parameters:
'SubmissionFileId',
'Message',
'ErrorCode',
'Action',
'PerfData',
'Secret',
'UUID'
GET requests are expected to contain the following parameters:
'Secret',
'UUID'
GET reponses deliver the following elements in the header:
'SubmissionFileId',
'Timeout',
'Action',
'PostRunValidation'
'''
try:
if request.method == 'GET':
secret = request.GET['Secret']
uuid = request.GET['UUID']
elif request.method == 'POST':
secret = request.POST['Secret']
uuid = request.POST['UUID']
except Exception as e:
logger.error("Error finding the neccessary data in the executor request: "+str(e))
raise PermissionDenied
if secret != settings.JOB_EXECUTOR_SECRET:
raise PermissionDenied
try:
logger.debug("Test machine is known, updating last contact timestamp")
machine = TestMachine.objects.get(host=uuid)
machine.last_contact = datetime.now()
machine.save()
except:
# ask for configuration of new execution hosts by returning the according action
logger.debug("Test machine is unknown, asking executor for configuration")
machine = TestMachine(host=uuid, last_contact=datetime.now())
machine.save()
response = HttpResponse()
response['Action'] = 'get_config'
response['MachineId'] = machine.pk
return response
if request.method == "GET":
subm = Submission.pending_student_tests.filter(assignment__in=machine.assignments.all()).all()
if len(subm) == 0:
logger.debug("No pending compile or validation jobs")
subm = Submission.pending_full_tests.filter(assignment__in=machine.assignments.all()).all()
if len(subm) == 0:
logger.debug("No pending full test jobs")
raise Http404
for sub in subm:
logger.debug("Got %u executor jobs"%(len(subm)))
assert (sub.file_upload) # must be given when the state model is correct
if (machine in sub.assignment.test_machines.all()):
# Machine is a candidate for this job
fetch_date = sub.get_fetch_date()
if fetch_date:
# This job was already fetched, check how long ago this happened
max_delay = timedelta(seconds=sub.assignment.attachment_test_timeout)
if fetch_date + max_delay < datetime.now():
logger.debug("Resetting executor fetch status for submission %u, due to timeout"%sub.pk)
# Stuff that has timed out
# we mark it as failed so that the user gets informed
# TODO: Late delivery for such a submission by the executor witll break everything
sub.clean_fetch_date()
if sub.state == Submission.TEST_COMPILE_PENDING:
sub.state = Submission.TEST_COMPILE_FAILED
sub.save_compile_result(machine, "Killed due to non-reaction on timeout signals. Please check your application for deadlocks or keyboard input.")
inform_student(sub, sub.state)
if sub.state == Submission.TEST_VALIDITY_PENDING:
sub.save_validation_result(machine, "Killed due to non-reaction on timeout signals. Please check your application for deadlocks or keyboard input.", None)
sub.state = Submission.TEST_VALIDITY_FAILED
inform_student(sub, sub.state)
if sub.state == Submission.TEST_FULL_PENDING:
sub.save_fulltest_result(machine, "Killed due to non-reaction on timeout signals. Student not informed, since this was the full test.", None)
sub.state = Submission.TEST_FULL_FAILED
sub.save()
continue
else:
logger.debug("Submission %u was already fetched, still waiting for it"%sub.pk)
else:
# Requesting machine fits, not fetched so far
# create HTTP response with file download
f = sub.file_upload.attachment
# on dev server, we sometimes have stale database entries
if not os.access(f.path, os.F_OK):
mail_managers('Warning: Missing file',
'Missing file on storage for submission file entry %u: %s' % (
sub.file_upload.pk, str(sub.file_upload.attachment)), fail_silently=True)
continue
response = HttpResponse(f, content_type='application/binary')
response['Content-Disposition'] = 'attachment; filename="%s"' % sub.file_upload.basename()
response['SubmissionFileId'] = str(sub.file_upload.pk)
response['SubmissionId'] = str(sub.pk)
response['Timeout'] = sub.assignment.attachment_test_timeout
if sub.state == Submission.TEST_COMPILE_PENDING:
response['Action'] = 'test_compile'
elif sub.state == Submission.TEST_VALIDITY_PENDING:
response['Action'] = 'test_validity'
response['PostRunValidation'] = sub.assignment.validity_test_url()
elif sub.state == Submission.TEST_FULL_PENDING or sub.state == Submission.CLOSED_TEST_FULL_PENDING:
response['Action'] = 'test_full'
response['PostRunValidation'] = sub.assignment.full_test_url()
else:
assert (False)
# store date of fetching for determining jobs stucked at the executor
sub.save_fetch_date()
# 'touch' submission so that it becomes sorted to the end of the queue if something goes wrong
sub.modified = datetime.now()
sub.save()
return response
else:
logger.debug("Requesting machine is not responsible for submission %u"%sub.pk)
# candidate submissions did not fit
raise Http404
elif request.method == "POST":
# first check if this is just configuration data, and not a job result
if request.POST['Action'] == 'get_config':
machine = TestMachine.objects.get(pk=int(request.POST['MachineId']))
machine.config = request.POST['Config']
machine.save()
return HttpResponse(status=201)
# executor.py is providing the results as POST parameters
sid = request.POST['SubmissionFileId']
perf_data = request.POST['PerfData'].strip()
submission_file = get_object_or_404(SubmissionFile, pk=sid)
sub = submission_file.submissions.all()[0]
error_code = int(request.POST['ErrorCode'])
if request.POST['Action'] == 'test_compile' and sub.state == Submission.TEST_COMPILE_PENDING:
sub.save_compile_result(machine, request.POST['Message'])
if error_code == 0:
if sub.assignment.attachment_test_validity:
sub.state = Submission.TEST_VALIDITY_PENDING
elif sub.assignment.attachment_test_full:
sub.state = Submission.TEST_FULL_PENDING
else:
sub.state = Submission.SUBMITTED_TESTED
inform_course_owner(request, sub)
else:
sub.state = Submission.TEST_COMPILE_FAILED
inform_student(sub, sub.state)
elif request.POST['Action'] == 'test_validity' and sub.state == Submission.TEST_VALIDITY_PENDING:
sub.save_validation_result(machine, request.POST['Message'], perf_data)
if error_code == 0:
if sub.assignment.attachment_test_full:
sub.state = Submission.TEST_FULL_PENDING
else:
sub.state = Submission.SUBMITTED_TESTED
inform_course_owner(request, sub)
else:
sub.state = Submission.TEST_VALIDITY_FAILED
inform_student(sub, sub.state)
elif request.POST['Action'] == 'test_full' and sub.state == Submission.TEST_FULL_PENDING:
sub.save_fulltest_result(machine, request.POST['Message'], perf_data)
if error_code == 0:
sub.state = Submission.SUBMITTED_TESTED
inform_course_owner(request, sub)
else:
sub.state = Submission.TEST_FULL_FAILED
# full tests may be performed several times and are meant to be a silent activity
# therefore, we send no mail to the student here
elif request.POST['Action'] == 'test_full' and sub.state == Submission.CLOSED_TEST_FULL_PENDING:
sub.save_fulltest_result(machine, request.POST['Message'], perf_data)
sub.state = Submission.CLOSED
# full tests may be performed several times and are meant to be a silent activity
# therefore, we send no mail to the student here
else:
msg = '''
Dear OpenSubmit administrator,
the executors returned some result, but this does not fit to the current submission state.
This is a strong indication for a bug in OpenSubmit - sorry for that.
The system will ignore the report from executor and mark the job as to be repeated.
Please report this on the project GitHub page for further investigation.
Submission ID: %u
Submission File ID reported by the executor: %u
Action reported by the executor: %s
Current state of the submission: %s (%s)
Message from the executor: %s
Error code from the executor: %u
'''%( sub.pk, submission_file.pk, request.POST['Action'],
sub.state_for_tutors(), sub.state,
request.POST['Message'], error_code )
mail_managers('Warning: Inconsistent job state', msg, fail_silently=True)
sub.clean_fetch_date()
sub.save()
return HttpResponse(status=201)
