def __init__(self):
# Attempt to load settings for the Mathworks servers
self.api_key = settings.MATHWORKS_API_KEY
# Fail immediately if settings are missing
# Check the configuration in test_env.json
# (see xqueue.test_settings for details)
assert(self.api_key is not None)
# Initialize as many response listeners as we have
# problems. A submission to a problem will go
# to a specific response listener.
self.response_listeners = []
self.clients = []
for i in range(Transaction.NUM_PROBLEMS):
# Create the response listener
# This is unique for our test, so we can capture
# responses only for the submissions we sent
listener = GradeResponseListener()
self.response_listeners.append(listener)
# Create a client that submits to this problem
# (configure it so the xqueue will send its response
# to this response listener)
client = XQueueTestClient(listener.port_num())
self.clients.append(client)
# Log in and store the session
self.session = self._log_in()
def setUp(self):
"""Set up the client and stubs to be used across tests."""
# Attempt to load settings for the Mathworks servers
self.api_key = settings.MATHWORKS_API_KEY
self.grader_url = settings.XQUEUES.get('matlab', None)
# Skip if the settings are missing
if self.api_key is None or self.grader_url is None:
raise SkipTest(
'You must specify an API key and URL for Mathworks in test_env.json'
)
# Create the response listener
# which listens for responses on an open port
self.response_listener = GradeResponseListener()
# Create the client (input submissions)
# and configure it to send messages
# that xqueue will send to our response listener
self.client = XQueueTestClient(self.response_listener.port_num())
# Create the user and make sure we are logged in
XQueueTestClient.create_user('test', '*****@*****.**', 'password')
self.client.login(username='******', password='******')
# Start up workers to pull messages from the queue
# and forward them to our grader
PassiveGraderStub.start_workers_for_grader_url(
MatlabGraderTest.QUEUE_NAME, self.grader_url)
def setUp(self):
"""Set up the client and stubs to be used across tests."""
# Attempt to load settings for the Mathworks servers
self.api_key = settings.MATHWORKS_API_KEY
self.grader_url = settings.XQUEUES.get('matlab', None)
# Skip if the settings are missing
if self.api_key is None or self.grader_url is None:
raise SkipTest('You must specify an API key and URL for Mathworks in test_env.json')
# Create the response listener
# which listens for responses on an open port
self.response_listener = GradeResponseListener()
# Create the client (input submissions)
# and configure it to send messages
# that xqueue will send to our response listener
self.client = XQueueTestClient(self.response_listener.port_num())
# Create the user and make sure we are logged in
XQueueTestClient.create_user('test', '*****@*****.**', 'password')
self.client.login(username='******', password='******')
# Start up workers to pull messages from the queue
# and forward them to our grader
PassiveGraderStub.start_workers_for_grader_url(MatlabGraderTest.QUEUE_NAME,
self.grader_url)
def setUp(self):
"""Set up the client and stubs to be used across tests."""
# Create the grader
self.grader = SimpleActiveGrader(ActiveGraderTest.QUEUE_NAME,
ActiveGraderTest.GRADER_RESPONSE)
# Create the response listener
# and configure it to receive messages on a local port
self.response_listener = GradeResponseListener()
# Create the client (input submissions)
# and configure it to send messages
# that will be sent back to our response listener
self.client = XQueueTestClient(self.response_listener.port_num())
# Create the user and make sure we are logged in
XQueueTestClient.create_user('test', '*****@*****.**', 'password')
self.client.login(username='******', password='******')
def setUp(self):
"""Set up the client and stubs to be used across tests."""
# Create the grader
self.grader = SimpleActiveGrader(ActiveGraderTest.QUEUE_NAME,
ActiveGraderTest.GRADER_RESPONSE)
# Create the response listener
# and configure it to receive messages on a local port
self.response_listener = GradeResponseListener()
# Create the client (input submissions)
# and configure it to send messages
# that will be sent back to our response listener
self.client = XQueueTestClient(self.response_listener.port_num())
# Create the user and make sure we are logged in
XQueueTestClient.create_user('test', '*****@*****.**', 'password')
self.client.login(username='******', password='******')
class MatlabGraderTest(TransactionTestCase):
"""Test that we can send messages to the xqueue
and receive a response from a Mathworks server
The test requires that the Mathworks end-point is set up
correctly in the settings:
* `XQUEUES` must contain an entry for matlab: `{'matlab': URL }`
* `MATLAB_API_KEY` must contain the API key to send the Mathworks servers.
You can specify these in test_env.json (see test_settings.py for details)
If the required settings cannot be loaded, the test will fail."""
# Choose a unique queue name to prevent conflicts in Jenkins
QUEUE_NAME = 'matlab_%s' % uuid4().hex
def setUp(self):
"""Set up the client and stubs to be used across tests."""
# Attempt to load settings for the Mathworks servers
self.api_key = settings.MATHWORKS_API_KEY
self.grader_url = settings.XQUEUES.get('matlab', None)
# Skip if the settings are missing
if self.api_key is None or self.grader_url is None:
raise SkipTest('You must specify an API key and URL for Mathworks in test_env.json')
# Create the response listener
# which listens for responses on an open port
self.response_listener = GradeResponseListener()
# Create the client (input submissions)
# and configure it to send messages
# that xqueue will send to our response listener
self.client = XQueueTestClient(self.response_listener.port_num())
# Create the user and make sure we are logged in
XQueueTestClient.create_user('test', '*****@*****.**', 'password')
self.client.login(username='******', password='******')
# Start up workers to pull messages from the queue
# and forward them to our grader
PassiveGraderStub.start_workers_for_grader_url(MatlabGraderTest.QUEUE_NAME,
self.grader_url)
def tearDown(self):
"""Stop each of the listening services to free up the ports"""
self.response_listener.stop()
# Stop the workers we started earlier
PassiveGraderStub.stop_workers()
# Delete the queue we created
PassiveGraderStub.delete_queue(MatlabGraderTest.QUEUE_NAME)
def test_matlab_check_correct(self):
response = self._submit_to_mathworks("assert(isequal(x,1))", "x=1")
expected = dedent("""
<div class='matlabResponse'><div style='white-space:pre' class='commandWindowOutput'>
x =
1
</div><ul></ul></div>
""").strip()
self.assertEqual(response.get('msg', None), expected)
self.assertEqual(response.get('correct', None), True)
self.assertEqual(response.get('score', None), 1)
def test_matlab_check_incorrect(self):
response = self._submit_to_mathworks("assert(isequal(x,1))", "x=5")
expected = dedent("""
<div class='matlabResponse'><div style='white-space:pre' class='commandWindowOutput'>
x =
5
</div><ul><li>Assertion failed.
</li></ul></div>
""").strip()
self.assertEqual(response.get('msg', None), expected)
self.assertEqual(response.get('correct', None), False)
self.assertEqual(response.get('score', None), 0)
def test_matlab_graph(self):
response = self._submit_to_mathworks("peaks;", "")
# The response message is usually very long,
# so we scan for the CSS tag instead.
response_msg = response.get('msg', '')
self.assertTrue('matlabFigures' in response_msg)
self.assertEqual(response.get('correct', None), True)
self.assertEqual(response.get('score', None), 1)
def test_matlab_invalid(self):
response = self._submit_to_mathworks("invalid", "x=5")
expected = dedent("""
<div class='matlabResponse'><div style='white-space:pre' class='commandWindowOutput'>
x =
5
</div><ul><li>Undefined function or variable 'invalid'.
</li></ul></div>
""").strip()
self.assertEqual(response.get('msg', None), expected)
self.assertEqual(response.get('correct', None), False)
self.assertEqual(response.get('score', None), 0)
def _submit_to_mathworks(self, matlab_code, student_input):
"""Assert that Mathworks servers provide the correct response.
`matlab_code`: Matlab code to be processed by external servers (string)
`student_input`: The student's response (string)
Returns the response from Mathworks (dict)"""
payload = "%%api_key=%s\n%%%%\n%s\n" % (self.api_key, matlab_code)
# Tell the xqueue to forward messages to the mathworks grader
# using our unique queue name
xqueue_settings = {MatlabGraderTest.QUEUE_NAME: self.grader_url}
with override_settings(XQUEUES=xqueue_settings):
# Send the XQueue a submission to be graded
submission = self.client.build_request(MatlabGraderTest.QUEUE_NAME,
grader_payload=payload,
student_response=student_input)
self.client.send_request(submission)
# Poll the response listener until we get a response
# or reach the timeout
def poll_func(listener):
return len(listener.get_grade_responses()) > 0
success = self.response_listener.block_until(poll_func,
sleep_time=0.5,
timeout=10.0)
# Check that we did not time out
self.assertTrue(success, 'Timed out waiting for response')
# Return the response
responses = self.response_listener.get_grade_responses()
xqueue_body = responses[0]['response']['xqueue_body']
return xqueue_body
class MatlabGraderTest(TransactionTestCase):
"""Test that we can send messages to the xqueue
and receive a response from a Mathworks server
The test requires that the Mathworks end-point is set up
correctly in the settings:
* `XQUEUES` must contain an entry for matlab: `{'matlab': URL }`
* `MATLAB_API_KEY` must contain the API key to send the Mathworks servers.
You can specify these in test_env.json (see test_settings.py for details)
If the required settings cannot be loaded, the test will fail."""
# Choose a unique queue name to prevent conflicts in Jenkins
QUEUE_NAME = 'matlab_%s' % uuid4().hex
def setUp(self):
"""Set up the client and stubs to be used across tests."""
# Attempt to load settings for the Mathworks servers
self.api_key = settings.MATHWORKS_API_KEY
self.grader_url = settings.XQUEUES.get('matlab', None)
# Skip if the settings are missing
if self.api_key is None or self.grader_url is None:
raise SkipTest(
'You must specify an API key and URL for Mathworks in test_env.json'
)
# Create the response listener
# which listens for responses on an open port
self.response_listener = GradeResponseListener()
# Create the client (input submissions)
# and configure it to send messages
# that xqueue will send to our response listener
self.client = XQueueTestClient(self.response_listener.port_num())
# Create the user and make sure we are logged in
XQueueTestClient.create_user('test', '*****@*****.**', 'password')
self.client.login(username='******', password='******')
# Start up workers to pull messages from the queue
# and forward them to our grader
PassiveGraderStub.start_workers_for_grader_url(
MatlabGraderTest.QUEUE_NAME, self.grader_url)
def tearDown(self):
"""Stop each of the listening services to free up the ports"""
self.response_listener.stop()
# Stop the workers we started earlier
PassiveGraderStub.stop_workers()
# Delete the queue we created
PassiveGraderStub.delete_queue(MatlabGraderTest.QUEUE_NAME)
def test_matlab_check_correct(self):
response = self._submit_to_mathworks("assert(isequal(x,1))", "x=1")
expected = dedent("""
<div class='matlabResponse'><div style='white-space:pre' class='commandWindowOutput'>
x =
1
</div><ul></ul></div>
""").strip()
self.assertEqual(response.get('msg', None), expected)
self.assertEqual(response.get('correct', None), True)
self.assertEqual(response.get('score', None), 1)
def test_matlab_check_incorrect(self):
response = self._submit_to_mathworks("assert(isequal(x,1))", "x=5")
expected = dedent("""
<div class='matlabResponse'><div style='white-space:pre' class='commandWindowOutput'>
x =
5
</div><ul><li>Assertion failed.
</li></ul></div>
""").strip()
self.assertEqual(response.get('msg', None), expected)
self.assertEqual(response.get('correct', None), False)
self.assertEqual(response.get('score', None), 0)
def test_matlab_graph(self):
response = self._submit_to_mathworks("peaks;", "")
# The response message is usually very long,
# so we scan for the CSS tag instead.
response_msg = response.get('msg', '')
self.assertTrue('matlabFigures' in response_msg)
self.assertEqual(response.get('correct', None), True)
self.assertEqual(response.get('score', None), 1)
def test_matlab_invalid(self):
response = self._submit_to_mathworks("invalid", "x=5")
expected = dedent("""
<div class='matlabResponse'><div style='white-space:pre' class='commandWindowOutput'>
x =
5
</div><ul><li>Undefined function or variable 'invalid'.
</li></ul></div>
""").strip()
self.assertEqual(response.get('msg', None), expected)
self.assertEqual(response.get('correct', None), False)
self.assertEqual(response.get('score', None), 0)
def _submit_to_mathworks(self, matlab_code, student_input):
"""Assert that Mathworks servers provide the correct response.
`matlab_code`: Matlab code to be processed by external servers (string)
`student_input`: The student's response (string)
Returns the response from Mathworks (dict)"""
payload = "%%api_key=%s\n%%%%\n%s\n" % (self.api_key, matlab_code)
# Tell the xqueue to forward messages to the mathworks grader
# using our unique queue name
xqueue_settings = {MatlabGraderTest.QUEUE_NAME: self.grader_url}
with override_settings(XQUEUES=xqueue_settings):
# Send the XQueue a submission to be graded
submission = self.client.build_request(
MatlabGraderTest.QUEUE_NAME,
grader_payload=payload,
student_response=student_input)
self.client.send_request(submission)
# Poll the response listener until we get a response
# or reach the timeout
def poll_func(listener):
return len(listener.get_grade_responses()) > 0
success = self.response_listener.block_until(poll_func,
sleep_time=0.5,
timeout=10.0)
# Check that we did not time out
self.assertTrue(success, 'Timed out waiting for response')
# Return the response
responses = self.response_listener.get_grade_responses()
xqueue_body = responses[0]['response']['xqueue_body']
return xqueue_body
class PassiveGraderTest(TransactionTestCase):
"""Test that we can send messages to the xqueue
and receive a response when using a "passive" external
grader (one that expects xqueue to send it submissions)"""
GRADER_RESPONSE = {'submission_data': 'test'}
# Choose a unique queue name to prevent conflicts
# in Jenkins
QUEUE_NAME = 'test_queue_%s' % uuid4().hex
def setUp(self):
"""Set up the client and stubs to be used across tests."""
# Create the grader
self.grader = SimplePassiveGrader(PassiveGraderTest.GRADER_RESPONSE)
# Create the response listener
# and configure it to receive messages on a local port
self.response_listener = GradeResponseListener()
# Create the client (input submissions)
# and configure it to send messages
# that will be sent back to our response listener
self.client = XQueueTestClient(self.response_listener.port_num())
# Create the user and make sure we are logged in
XQueueTestClient.create_user('test', '*****@*****.**', 'password')
self.client.login(username='******', password='******')
# Start up workers to pull messages from the queue
# and forward them to our grader
self.grader.start_workers(PassiveGraderTest.QUEUE_NAME)
def tearDown(self):
"""Stop each of the listening services to free up the ports"""
self.grader.stop()
self.response_listener.stop()
# Stop the workers we started earlier
SimplePassiveGrader.stop_workers()
# Delete the queue we created
SimplePassiveGrader.delete_queue(PassiveGraderTest.QUEUE_NAME)
def test_submission(self):
"""Submit a single response to the XQueue and check that
we get the expected response."""
payload = {'test': 'test'}
student_input = 'test response'
# Tell the xqueue to forward messages to our grader
xqueue_settings = {PassiveGraderTest.QUEUE_NAME: self.grader.grader_url()}
with override_settings(XQUEUES=xqueue_settings):
# Send the XQueue a submission to be graded
submission = self.client.build_request(PassiveGraderTest.QUEUE_NAME,
grader_payload=payload,
student_response=student_input)
self.client.send_request(submission)
# Poll the response listener until we get a response
# or reach the timeout
def poll_func(listener):
return len(listener.get_grade_responses()) > 0
success = self.response_listener.block_until(poll_func,
sleep_time=0.5,
timeout=4.0)
# Check that we did not time out
self.assertTrue(success)
# Check the response matches what we expect
responses = self.response_listener.get_grade_responses()
xqueue_body = responses[0]['response']['xqueue_body']
self.assertEqual(PassiveGraderTest.GRADER_RESPONSE, xqueue_body)
class PassiveGraderTest(unittest.TestCase):
"""Test that we can send messages to the xqueue
and receive a response when using a "passive" external
grader (one that expects xqueue to send it submissions)"""
GRADER_RESPONSE = {'submission_data': 'test'}
# Choose a unique queue name to prevent conflicts
# in Jenkins
QUEUE_NAME = 'test_queue_%s' % uuid4().hex
def setUp(self):
"""Set up the client and stubs to be used across tests."""
# Create the grader
self.grader = SimplePassiveGrader(PassiveGraderTest.GRADER_RESPONSE)
# Create the response listener
# and configure it to receive messages on a local port
self.response_listener = GradeResponseListener()
# Create the client (input submissions)
# and configure it to send messages
# that will be sent back to our response listener
self.client = XQueueTestClient(self.response_listener.port_num())
# Create the user and make sure we are logged in
XQueueTestClient.create_user('test', '*****@*****.**', 'password')
self.client.login(username='******', password='******')
# Start up workers to pull messages from the queue
# and forward them to our grader
self.grader.start_workers(PassiveGraderTest.QUEUE_NAME)
def tearDown(self):
"""Stop each of the listening services to free up the ports"""
self.grader.stop()
self.response_listener.stop()
# Stop the workers we started earlier
SimplePassiveGrader.stop_workers()
# Delete the queue we created
SimplePassiveGrader.delete_queue(PassiveGraderTest.QUEUE_NAME)
def test_submission(self):
"""Submit a single response to the XQueue and check that
we get the expected response."""
payload = {'test': 'test'}
student_input = 'test response'
# Tell the xqueue to forward messages to our grader
xqueue_settings = {
PassiveGraderTest.QUEUE_NAME: self.grader.grader_url()
}
with override_settings(XQUEUES=xqueue_settings):
# Send the XQueue a submission to be graded
submission = self.client.build_request(
PassiveGraderTest.QUEUE_NAME,
grader_payload=payload,
student_response=student_input)
self.client.send_request(submission)
# Poll the response listener until we get a response
# or reach the timeout
poll_func = lambda listener: len(listener.get_grade_responses()
) > 0
success = self.response_listener.block_until(poll_func,
sleep_time=0.5,
timeout=4.0)
# Check that we did not time out
self.assertTrue(success)
# Check the response matches what we expect
responses = self.response_listener.get_grade_responses()
xqueue_body = responses[0]['response']['xqueue_body']
self.assertEqual(PassiveGraderTest.GRADER_RESPONSE, xqueue_body)
class ActiveGraderTest(TransactionTestCase):
"""Test that we can send messages to the xqueue
and receive a response when using an "active" external
grader (one that polls the XQueue and pushes responses using
a REST-like interface)
"""
GRADER_RESPONSE = {'submission_data': 'test'}
# Choose a unique queue name to prevent conflicts
# in Jenkins
QUEUE_NAME = 'test_queue_%s' % uuid4().hex
def setUp(self):
"""Set up the client and stubs to be used across tests."""
# Create the grader
self.grader = SimpleActiveGrader(ActiveGraderTest.QUEUE_NAME,
ActiveGraderTest.GRADER_RESPONSE)
# Create the response listener
# and configure it to receive messages on a local port
self.response_listener = GradeResponseListener()
# Create the client (input submissions)
# and configure it to send messages
# that will be sent back to our response listener
self.client = XQueueTestClient(self.response_listener.port_num())
# Create the user and make sure we are logged in
XQueueTestClient.create_user('test', '*****@*****.**', 'password')
self.client.login(username='******', password='******')
# Since the ActiveGraderStub is polling the XQueue,
# we do NOT need to start any workers (consumers)
# that pull messages from the XQueue and pass them on
def tearDown(self):
"""Stop each of the listening services to free up the ports"""
self.grader.stop()
self.response_listener.stop()
def test_submission(self):
"""Submit a single response to the XQueue and check that
we get the expected response."""
payload = {'test': 'test'}
student_input = 'test response'
# Add our queue to the available queues
# Otherwise, XQueue will not process our messages
# We set the callback URL to None because XQueue does not
# need to forward the messages; instead, our ActiveGrader
# polls for them
xqueue_settings = settings.XQUEUES
xqueue_settings[ActiveGraderTest.QUEUE_NAME] = None
with override_settings(XQUEUES=xqueue_settings):
# Send the XQueue a submission to be graded
submission = self.client.build_request(
ActiveGraderTest.QUEUE_NAME,
grader_payload=payload,
student_response=student_input)
self.client.send_request(submission)
# Poll the response listener until we get a response
# or reach the timeout
def poll_func(listener):
return len(listener.get_grade_responses()) > 0
success = self.response_listener.block_until(poll_func,
sleep_time=0.5,
timeout=4.0)
# Check that we did not time out
self.assertTrue(success)
# Check the response matches what we expect
responses = self.response_listener.get_grade_responses()
xqueue_body = responses[0]['response']['xqueue_body']
self.assertEqual(ActiveGraderTest.GRADER_RESPONSE, xqueue_body)
class ActiveGraderTest(TransactionTestCase):
"""Test that we can send messages to the xqueue
and receive a response when using an "active" external
grader (one that polls the XQueue and pushes responses using
a REST-like interface)
"""
GRADER_RESPONSE = {'submission_data': 'test'}
# Choose a unique queue name to prevent conflicts
# in Jenkins
QUEUE_NAME = 'test_queue_%s' % uuid4().hex
def setUp(self):
"""Set up the client and stubs to be used across tests."""
# Create the grader
self.grader = SimpleActiveGrader(ActiveGraderTest.QUEUE_NAME,
ActiveGraderTest.GRADER_RESPONSE)
# Create the response listener
# and configure it to receive messages on a local port
self.response_listener = GradeResponseListener()
# Create the client (input submissions)
# and configure it to send messages
# that will be sent back to our response listener
self.client = XQueueTestClient(self.response_listener.port_num())
# Create the user and make sure we are logged in
XQueueTestClient.create_user('test', '*****@*****.**', 'password')
self.client.login(username='******', password='******')
# Since the ActiveGraderStub is polling the XQueue,
# we do NOT need to start any workers (consumers)
# that pull messages from the XQueue and pass them on
def tearDown(self):
"""Stop each of the listening services to free up the ports"""
self.grader.stop()
self.response_listener.stop()
# Delete the queue we created
SimpleActiveGrader.delete_queue(ActiveGraderTest.QUEUE_NAME)
def test_submission(self):
"""Submit a single response to the XQueue and check that
we get the expected response."""
payload = {'test': 'test'}
student_input = 'test response'
# Add our queue to the available queues
# Otherwise, XQueue will not process our messages
# We set the callback URL to None because XQueue does not
# need to forward the messages; instead, our ActiveGrader
# polls for them
xqueue_settings = settings.XQUEUES
xqueue_settings[ActiveGraderTest.QUEUE_NAME] = None
with override_settings(XQUEUES=xqueue_settings):
# Send the XQueue a submission to be graded
submission = self.client.build_request(ActiveGraderTest.QUEUE_NAME,
grader_payload=payload,
student_response=student_input)
self.client.send_request(submission)
# Poll the response listener until we get a response
# or reach the timeout
def poll_func(listener):
return len(listener.get_grade_responses()) > 0
success = self.response_listener.block_until(poll_func,
sleep_time=0.5,
timeout=4.0)
# Check that we did not time out
self.assertTrue(success)
# Check the response matches what we expect
responses = self.response_listener.get_grade_responses()
xqueue_body = responses[0]['response']['xqueue_body']
self.assertEqual(ActiveGraderTest.GRADER_RESPONSE, xqueue_body)
