def test_twitter_search_gets_processed():
"""Ensures the data can be loaded from twitter and stored as a raw source
Run one search query
Run worker
Check two tweets in raw source
Run worker
Check the pixels have been averaged out
"""
with Connection(connection=redis_db):
source_queue = Queue(name='source')
process_queue = Queue(name='process')
query = 'Test Query'
size = 2
twitter.search(input=query, size=size)
worker = SimpleWorker([source_queue])
worker.work(burst=True)
assert len(Tweet.keys()) == size
worker = SimpleWorker([process_queue])
worker.work(burst=True)
#
# assert len(Tweet.keys()) == size
def test_func__reset_password__ok__nominal_case(self):
uapi = UserApi(
current_user=None,
session=self.session,
config=self.app_config,
)
current_user = uapi.get_one_by_email('*****@*****.**')
uapi.reset_password_notification(current_user, do_save=True)
transaction.commit()
# Send mail async from redis queue
redis = get_redis_connection(
self.app_config
)
queue = get_rq_queue(
redis,
'mail_sender',
)
worker = SimpleWorker([queue], connection=queue.connection)
worker.work(burst=True)
# check mail received
response = self.get_mailhog_mails()
headers = response[0]['Content']['Headers']
assert headers['From'][0] == 'Tracim Notifications <test_user_from+0@localhost>' # nopep8
assert headers['To'][0] == 'Global manager <*****@*****.**>'
assert headers['Subject'][0] == '[TRACIM] A password reset has been requested'
def test_coverage_summary_by_changeset(coverage_builds):
from rq import Queue
from codecoverage_backend import api
from tests.conftest import mock_coverage_by_changeset_job_success
# patch the queue to be sync to allow it run without workers. http://python-rq.org/docs/testing/
with mock.patch('codecoverage_backend.api.q', Queue(connection=FakeStrictRedis())) as q:
# patch the mock_coverage_by_changeset
with mock.patch('codecoverage_backend.api.coverage_by_changeset_job', mock_coverage_by_changeset_job_success):
# Get changeset coverage information
for changeset, expected in coverage_builds['summary'].items():
result, code = api.coverage_summary_by_changeset(changeset)
assert code == 202
# test that in the case of exception it will return 500
result, code = api.coverage_summary_by_changeset('mozilla test changeset')
assert code == 202
# run simple worker to run all tasks
w = SimpleWorker([q], connection=q.connection)
w.work(burst=True)
# Everything should be 200 now
for changeset, expected in coverage_builds['summary'].items():
result, code = api.coverage_summary_by_changeset(changeset)
assert result == expected
assert code == 200
# except the incorrect changeset, should be 500
result, code = api.coverage_summary_by_changeset('mozilla test changeset')
assert code == 500
def test_func__create_new_content_with_notification__ok__nominal_case(self):
uapi = UserApi(
current_user=None,
session=self.session,
config=self.app_config,
)
current_user = uapi.get_one_by_email('*****@*****.**')
# Create new user with notification enabled on w1 workspace
wapi = WorkspaceApi(
current_user=current_user,
session=self.session,
config=self.app_config,
)
workspace = wapi.get_one_by_label('Recipes')
user = uapi.get_one_by_email('*****@*****.**')
wapi.enable_notifications(user, workspace)
api = ContentApi(
current_user=user,
session=self.session,
config=self.app_config,
)
item = api.create(
content_type_list.Folder.slug,
workspace,
None,
'parent',
do_save=True,
do_notify=False,
)
item2 = api.create(
content_type_list.File.slug,
workspace,
item,
'file1',
do_save=True,
do_notify=True,
)
# Send mail async from redis queue
redis = get_redis_connection(
self.app_config
)
queue = get_rq_queue(
redis,
'mail_sender',
)
worker = SimpleWorker([queue], connection=queue.connection)
worker.work(burst=True)
# check mail received
response = self.get_mailhog_mails()
headers = response[0]['Content']['Headers']
assert headers['From'][0] == '"Bob i. via Tracim" <test_user_from+3@localhost>' # nopep8
assert headers['To'][0] == 'Global manager <*****@*****.**>'
assert headers['Subject'][0] == '[TRACIM] [Recipes] file1 (Open)'
assert headers['References'][0] == 'test_user_refs+22@localhost'
assert headers['Reply-to'][0] == '"Bob i. & all members of Recipes" <test_user_reply+22@localhost>' # nopep8
def test_work_via_simpleworker(self):
"""Worker processes work, with forking disabled,
then returns."""
fooq, barq = Queue("foo"), Queue("bar")
w = SimpleWorker([fooq, barq])
self.assertEquals(w.work(burst=True), False, "Did not expect any work on the queue.")
job = fooq.enqueue(say_pid)
self.assertEquals(w.work(burst=True), True, "Expected at least some work done.")
self.assertEquals(job.result, os.getpid(), "PID mismatch, fork() is not supposed to happen here")
def test_simpleworker_heartbeat_ttl(self):
"""SimpleWorker's key must last longer than job.timeout when working"""
queue = Queue('foo')
worker = SimpleWorker([queue])
job_timeout = 300
job = queue.enqueue(save_key_ttl, worker.key, job_timeout=job_timeout)
worker.work(burst=True)
job.refresh()
self.assertGreater(job.meta['ttl'], job_timeout)
def process_worker_jobs():
# We need to do this while testing to avoid strange errors on Circle.
#
# See:
#
# http://python-rq.org/docs/testing/
# https://github.com/ui/django-rq/issues/123
queue = django_rq.get_queue()
worker = SimpleWorker([queue], connection=queue.connection)
worker.work(burst=True)
def test_work_via_simpleworker(self):
"""Worker processes work, with forking disabled,
then returns."""
fooq = Queue('foo', connection=self.conn)
barq = Queue('bar', connection=self.conn)
w = SimpleWorker([fooq, barq], connection=self.conn)
self.assertEqual(w.work(burst=True), False,
'Did not expect any work on the queue.')
job = fooq.enqueue(say_pid)
self.assertEqual(w.work(burst=True), True,
'Expected at least some work done.')
self.assertEqual(job.result, os.getpid(),
'PID mismatch, fork() is not supposed to happen here')
def test_microservice(self):
"""Test the microservice."""
redis_conn.flushall()
req_1 = self.app.post('/put', data={'int': 1})
req_1_as_json = json.loads(req_1.data)
assert req_1_as_json['integer_received'] == 1
req_2 = self.app.post('/put', data={'int': 1})
del req_2
time.sleep(1) # Staggering requests
req_3 = self.app.post('/put', data={'int': 4})
del req_3
req_4 = self.app.post('/put', data={'int': 6})
del req_4
time.sleep(1) # Staggering requests
req_5 = self.app.post('/put', data={'int': 8})
del req_5
req_6 = self.app.post('/put', data={'int': 10})
del req_6
median_req = self.app.get('/median')
median_as_dict = json.loads(median_req.data)
median_not_finished_job_req = self.app.get(
'/tasks/{}'.format(median_as_dict['task_id'])
)
assert median_not_finished_job_req.status_code == 202
not_finished_as_dict = json.loads(median_not_finished_job_req.data)
assert not_finished_as_dict['message'] == (
"Still processing..."
)
time.sleep(1)
with Connection(redis_conn):
queue = Queue(connection=redis_conn)
worker = SimpleWorker([queue])
worker.work(burst=True)
median_job_req = self.app.get(
'/tasks/{}'.format(median_as_dict['task_id'])
)
results_as_dict = json.loads(median_job_req.data)
assert results_as_dict['task_results']['result'] == 5
def test_func__create_user_with_mail_notification__ok__nominal_case(self):
api = UserApi(
current_user=None,
session=self.session,
config=self.app_config,
)
u = api.create_user(
email='bob@bob',
password='******',
name='bob',
timezone='+2',
do_save=True,
do_notify=True,
)
assert u is not None
assert u.email == "bob@bob"
assert u.validate_password('password')
assert u.display_name == 'bob'
assert u.timezone == '+2'
# Send mail async from redis queue
redis = get_redis_connection(
self.app_config
)
queue = get_rq_queue(
redis,
'mail_sender',
)
worker = SimpleWorker([queue], connection=queue.connection)
worker.work(burst=True)
# check mail received
response = self.get_mailhog_mails()
headers = response[0]['Content']['Headers']
assert headers['From'][0] == 'Tracim Notifications <test_user_from+0@localhost>' # nopep8
assert headers['To'][0] == 'bob <bob@bob>'
assert headers['Subject'][0] == '[TRACIM] Created account'
def test_run_job1(client):
"""Test running a new job for a task"""
with client.application.app_context():
app = client.application
app.redis.flushall()
task_id = str(uuid4())
t = Task.create_task(task_id)
j = t.create_job()
job_id = j.job_id
t.save()
exec_mock = MagicMock()
exec_mock.validate_max_running_executions.return_value = True
client.application.executor = exec_mock
queue = Queue("jobs", is_async=False, connection=client.application.redis)
result = queue.enqueue(job_mod.run_job, t.task_id, job_id, "image", "command")
worker = SimpleWorker([queue], connection=queue.connection)
worker.work(burst=True)
t.reload()
expect(t.jobs).to_length(1)
job = t.jobs[0]
expect(job.executions).to_length(1)
execution = job.executions[0]
expect(execution.image).to_equal("image")
expect(execution.command).to_equal("command")
hash_key = f"rq:job:{result.id}"
res = app.redis.exists(hash_key)
expect(res).to_be_true()
res = app.redis.hget(hash_key, "status")
expect(res).to_equal("finished")
res = app.redis.hexists(hash_key, "data")
expect(res).to_be_true()
keys = app.redis.keys()
next_job_id = [
key
for key in keys
if key.decode("utf-8").startswith("rq:job")
and not key.decode("utf-8").endswith(result.id)
]
expect(next_job_id).to_length(1)
next_job_id = next_job_id[0]
res = app.redis.exists(next_job_id)
expect(res).to_be_true()
res = app.redis.hget(next_job_id, "status")
expect(res).to_equal("queued")
res = app.redis.hexists(next_job_id, "data")
expect(res).to_be_true()
res = app.redis.hget(next_job_id, "origin")
expect(res).to_equal("monitor")
res = app.redis.hget(next_job_id, "description")
expect(res).to_equal(
f"fastlane.worker.job.monitor_job('{task_id}', '{job_id}', '{execution.execution_id}')"
)
res = app.redis.hget(next_job_id, "timeout")
expect(res).to_equal("-1")
t.reload()
expect(t.jobs[0].executions[0].status).to_equal(JobExecution.Status.running)
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import redis
from rq import Connection
from rq import Queue
from rq import SimpleWorker
from codecoverage_backend import secrets
conn = redis.from_url(secrets.REDIS_URL)
def exc_handler(job, *exc_info):
job.cleanup(ttl=3600)
if __name__ == '__main__':
with Connection(conn):
worker = SimpleWorker(map(Queue, ['default']), exception_handlers=[])
worker.push_exc_handler(exc_handler)
worker.push_exc_handler(worker.move_to_failed_queue)
worker.work()
from diskover import listen, version, config
from rq import SimpleWorker, Connection
from redis import exceptions
from datetime import datetime
import diskover_bot_module
from diskover_bot_module import redis_conn
if __name__ == "__main__":
# parse cli arguments into cliargs dictionary
cliargs_bot = vars(diskover_bot_module.parse_cliargs_bot())
print("""\033[31m
___ _ ____ _ _ ____ _ _ ____ ____ ;
|__> | ==== |-:_ [__] \/ |=== |--< ["]
____ ____ ____ _ _ _ ___ ____ ___ /[_]\\
|___ |--< |--| |/\| |___ |==] [__] | ] [ v%s
Redis RQ worker bot for diskover crawler
Crawling all your stuff.
\033[0m""" % (version))
with Connection(redis_conn):
w = SimpleWorker(listen)
if cliargs_bot['burst']:
w.work(burst=True, logging_level=cliargs_bot['loglevel'])
else:
w.work(logging_level=cliargs_bot['loglevel'])
class ViewTests(flask_testing.TestCase):
def create_app(self):
app = views.app
app.config['TESTING'] = True
app.config['DEBUG'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/test.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
return app
def setUp(self):
views.db.init_app(self.app)
with self.app.test_request_context():
views.db.create_all()
self.queue = Queue(connection=fakeredis.FakeStrictRedis())
self.worker = SimpleWorker([self.queue],
connection=self.queue.connection)
def tearDown(self):
views.db.session.remove()
views.db.drop_all()
@classmethod
def setUpClass(cls):
logging.disable(logging.CRITICAL)
@classmethod
def tearDownClass(cls):
logging.disable(logging.NOTSET)
def test_check_valid_user_no_canvas_user_id(self, m):
@views.check_valid_user
def test_func():
pass # pragma: no cover
response = test_func()
self.assert_template_used('error.html')
self.assertIn('Not allowed!', response)
def test_check_valid_user_no_lti_logged_in(self, m):
session['canvas_user_id'] = 1234
@views.check_valid_user
def test_func():
pass # pragma: no cover
response = test_func()
self.assert_template_used('error.html')
self.assertIn('Not allowed!', response)
def test_check_valid_user_no_course_id(self, m):
session['canvas_user_id'] = 1234
session['lti_logged_in'] = True
@views.check_valid_user
def test_func():
pass # pragma: no cover
response = test_func()
self.assert_template_used('error.html')
self.assertIn('No course_id provided.', response)
def test_check_valid_user_is_admin(self, m):
session['canvas_user_id'] = 1234
session['lti_logged_in'] = True
session['is_admin'] = True
@views.check_valid_user
def test_func(**kwargs):
return 'Course ID: {}'.format(kwargs.get('course_id'))
response = test_func(course_id=1)
self.assertEqual('Course ID: 1', response)
def test_check_valid_user_no_enrollments(self, m):
m.register_uri('GET', '/api/v1/courses/1/enrollments', json=[])
session['canvas_user_id'] = 1234
session['lti_logged_in'] = True
@views.check_valid_user
def test_func(**kwargs):
pass # pragma: no cover
response = test_func(course_id=1)
self.assert_template_used('error.html')
self.assertIn(
'You are not enrolled in this course as a Teacher, TA, or Designer.',
response)
def test_check_valid_user_success(self, m):
m.register_uri('GET',
'/api/v1/courses/1/enrollments',
json=[{
'id': 1,
'course_id': 1,
'user_id': 1234,
'role': 'TeacherEnrollment'
}])
session['canvas_user_id'] = 1234
session['lti_logged_in'] = True
@views.check_valid_user
def test_func(**kwargs):
return 'Course ID: {}'.format(kwargs.get('course_id'))
response = test_func(course_id=1)
self.assertEqual('Course ID: 1', response)
def test_index(self, m):
response = self.client.get('/')
self.assertEqual(response.data,
"Please contact your System Administrator.")
def test_xml(self, m):
response = self.client.get('/lti.xml')
self.assert_200(response)
self.assert_template_used('lti.xml')
self.assertIn('application/xml', response.content_type)
self.assert_context('tool_id', config.LTI_TOOL_ID)
self.assertIn(url_for('lti_tool'), response.data)
def test_quiz(self, m):
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
response = self.client.get('/quiz/{}/'.format(course_id))
self.assert_200(response)
self.assert_template_used('userselect.html')
self.assertEqual(self.get_context_variable('course_id'),
str(course_id))
self.assertEqual(self.get_context_variable('current_page_number'), 1)
def test_update_background_no_json(self, m):
from views import update_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
job = self.queue.enqueue_call(func=update_background,
args=(course_id, None))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
job_result = job.result
meta_keys = ['status', 'status_msg', 'percent', 'error']
self.assertTrue(all(key in job_result for key in meta_keys))
self.assertEqual(job_result['status'], 'failed')
self.assertTrue(job_result['error'])
self.assertEqual(job_result['status_msg'], 'Invalid Request')
def test_update_background_no_course(self, m):
from views import update_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
m.register_uri('GET', '/api/v1/courses/1', status_code=404)
job = self.queue.enqueue_call(func=update_background,
args=(course_id, {
'percent': '200',
'user_ids': ['11', '12']
}))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
job_result = job.result
meta_keys = ['status', 'status_msg', 'percent', 'error']
self.assertTrue(all(key in job_result for key in meta_keys))
self.assertEqual(job_result['status'], 'failed')
self.assertTrue(job_result['error'])
self.assertEqual(job_result['status_msg'], 'Course not found.')
def test_update_background_no_percent(self, m):
from views import update_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
m.register_uri('GET',
'/api/v1/courses/1',
json={
'id': 1,
'name': 'Example Course'
})
job = self.queue.enqueue_call(func=update_background,
args=(course_id, {
'user_ids': ['11', '12']
}))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
job_result = job.result
meta_keys = ['status', 'status_msg', 'percent', 'error']
self.assertTrue(all(key in job_result for key in meta_keys))
self.assertEqual(job_result['status'], 'failed')
self.assertTrue(job_result['error'])
self.assertEqual(job_result['status_msg'], '`percent` field required.')
def test_update_background_refresh_error(self, m):
from views import update_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
m.register_uri('GET',
'/api/v1/courses/1',
json={
'id': 1,
'name': 'Example Course'
})
m.register_uri('GET',
'/api/v1/courses/1/quizzes',
json=[{
'id': 4,
'title': 'Quiz 4',
'time_limit': 10
}, {
'id': 5,
'title': 'Quiz 5',
'time_limit': 30
}])
m.register_uri('GET',
'/api/v1/courses/1/users/11',
json={
'id': 11,
'sortable_name': 'Joe Smyth',
'enrollments': [{
'type': 'StudentEnrollment'
}]
})
m.register_uri('GET',
'/api/v1/courses/1/users/12',
json={
'id': 12,
'sortable_name': 'Jack Smith',
'enrollments': [{
'type': 'StudentEnrollment'
}]
})
m.register_uri('POST',
'/api/v1/courses/1/quizzes/4/extensions',
status_code=404)
course = Course(course_id, course_name='Example Course')
views.db.session.add(course)
user = User(11, sortable_name='Joe Smyth')
views.db.session.add(user)
user2 = User(12, sortable_name='Jack Smith')
views.db.session.add(user2)
views.db.session.commit()
ext = Extension(course.id, user.id)
views.db.session.add(ext)
ext2 = Extension(course.id, user2.id)
views.db.session.add(ext2)
views.db.session.commit()
job = self.queue.enqueue_call(func=update_background,
args=(course_id, {
'percent': '200',
'user_ids': ['11', '12']
}))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
job_result = job.result
meta_keys = ['status', 'status_msg', 'percent', 'error']
self.assertTrue(all(key in job_result for key in meta_keys))
self.assertEqual(job_result['status'], 'failed')
self.assertTrue(job_result['error'])
self.assertEqual(
job_result['status_msg'],
'Error creating extension for quiz #4. Canvas status code: 404')
def test_update_background_no_quizzes(self, m):
from views import update_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
m.register_uri('GET',
'/api/v1/courses/1',
json={
'id': 1,
'name': 'Example Course'
})
m.register_uri('GET', '/api/v1/courses/1/quizzes', json=[])
m.register_uri('GET',
'/api/v1/courses/1/users/11',
json={
'id': 11,
'sortable_name': 'Joe Smyth'
})
m.register_uri('GET',
'/api/v1/courses/1/users/12',
json={
'id': 12,
'sortable_name': 'Jack Smith'
})
course = Course(course_id, course_name='Example Course')
views.db.session.add(course)
user = User(11, sortable_name='Joe Smyth')
views.db.session.add(user)
user2 = User(12, sortable_name='Jack Smith')
views.db.session.add(user2)
views.db.session.commit()
ext = Extension(course.id, user.id)
views.db.session.add(ext)
ext2 = Extension(course.id, user2.id)
views.db.session.add(ext2)
views.db.session.commit()
job = self.queue.enqueue_call(func=update_background,
args=(course_id, {
'percent': '200',
'user_ids': ['11', '12']
}))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
job_result = job.result
meta_keys = ['status', 'status_msg', 'percent', 'error']
self.assertTrue(all(key in job_result for key in meta_keys))
self.assertEqual(job_result['status'], 'failed')
self.assertTrue(job_result['error'])
self.assertEqual(job_result['status_msg'],
'Sorry, there are no quizzes for this course.')
def test_update_background_extension_error(self, m):
from views import update_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
m.register_uri('GET',
'/api/v1/courses/1',
json={
'id': 1,
'name': 'Example Course'
})
m.register_uri('GET',
'/api/v1/courses/1/quizzes',
json=[{
'id': 4,
'title': 'Quiz 4',
'time_limit': 10
}, {
'id': 5,
'title': 'Quiz 5',
'time_limit': 30
}, {
'id': 6,
'title': 'Quiz 6',
'time_limit': None
}, {
'id': 7,
'title': 'Quiz 7'
}])
m.register_uri('GET',
'/api/v1/courses/1/users/11',
json={
'id': 11,
'sortable_name': 'Joe Smyth'
})
m.register_uri('GET', '/api/v1/courses/1/users/12', status_code=404)
m.register_uri('GET',
'/api/v1/courses/1/users/13',
json={
'id': 13,
'sortable_name': 'Jack Smith',
'enrollments': [{
'type': 'StudentEnrollment'
}]
})
m.register_uri('POST',
'/api/v1/courses/1/quizzes/4/extensions',
status_code=404)
m.register_uri('POST',
'/api/v1/courses/1/quizzes/5/extensions',
status_code=200)
course = Course(course_id, course_name='Example Course')
views.db.session.add(course)
user = User(11, sortable_name='Joe Smyth')
views.db.session.add(user)
user2 = User(13, sortable_name='Jack Smith')
views.db.session.add(user2)
views.db.session.commit()
ext = Extension(course.id, user.id)
views.db.session.add(ext)
ext2 = Extension(course.id, user2.id)
views.db.session.add(ext2)
views.db.session.commit()
job = self.queue.enqueue_call(func=update_background,
args=(course_id, {
'percent': '200',
'user_ids': ['11', '12', '13']
}))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
job_result = job.result
meta_keys = ['status', 'status_msg', 'percent', 'error']
self.assertTrue(all(key in job_result for key in meta_keys))
self.assertEqual(job_result['status'], 'failed')
self.assertTrue(job_result['error'])
self.assertEqual(
job_result['status_msg'],
'Error creating extension for quiz #4. Canvas status code: 404')
def test_update_background(self, m):
from views import update_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
m.register_uri('GET',
'/api/v1/courses/1',
json={
'id': 1,
'name': 'Example Course'
})
m.register_uri('GET',
'/api/v1/courses/1/quizzes',
json=[{
'id': 4,
'title': 'Quiz 4',
'time_limit': 10
}, {
'id': 5,
'title': 'Quiz 5',
'time_limit': 30
}, {
'id': 6,
'title': 'Quiz 6',
'time_limit': None
}, {
'id': 7,
'title': 'Quiz 7'
}])
m.register_uri('GET',
'/api/v1/courses/1/users/11',
json={
'id': 11,
'sortable_name': 'Joe Smyth'
})
m.register_uri('GET', '/api/v1/courses/1/users/12', status_code=404)
m.register_uri('GET',
'/api/v1/courses/1/users/13',
json={
'id': 13,
'sortable_name': 'Jack Smith'
})
m.register_uri('POST',
'/api/v1/courses/1/quizzes/4/extensions',
status_code=200)
m.register_uri('POST',
'/api/v1/courses/1/quizzes/5/extensions',
status_code=200)
course = Course(course_id, course_name='Example Course')
views.db.session.add(course)
user = User(11, sortable_name='Joe Smyth')
views.db.session.add(user)
user2 = User(13, sortable_name='Jack Smith')
views.db.session.add(user2)
views.db.session.commit()
ext = Extension(course.id, user.id)
views.db.session.add(ext)
ext2 = Extension(course.id, user2.id)
views.db.session.add(ext2)
views.db.session.commit()
job = self.queue.enqueue_call(func=update_background,
args=(course_id, {
'percent': '200',
'user_ids': ['11', '12', '13']
}))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
job_result = job.result
meta_keys = ['status', 'status_msg', 'percent', 'error']
self.assertTrue(all(key in job_result for key in meta_keys))
self.assertEqual(job_result['status'], 'complete')
self.assertFalse(job_result['error'])
self.assertEqual(job_result['percent'], 100)
self.assertEqual(
job_result['status_msg'],
('Success! 2 quizzes have been updated for 3 student(s) to have '
'200% time. 2 quizzes have no time limit and were left unchanged.'
))
def test_refresh_background_no_course(self, m):
from views import refresh_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
m.register_uri('GET', '/api/v1/courses/1', status_code=404)
job = self.queue.enqueue_call(func=refresh_background,
args=(course_id, ))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
job_result = job.result
meta_keys = ['status', 'status_msg', 'percent', 'error']
self.assertTrue(all(key in job_result for key in meta_keys))
self.assertEqual(job_result['status'], 'failed')
self.assertEqual(job_result['status_msg'], 'Course not found.')
self.assertEqual(job_result['percent'], 0)
self.assertTrue(job_result['error'])
def test_refresh_background_no_missing_quizzes(self, m):
from views import refresh_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
m.register_uri('GET',
'/api/v1/courses/1',
json={
'id': 1,
'name': 'Example Course'
})
m.register_uri('GET',
'/api/v1/courses/1/quizzes',
json=[{
'id': 3,
'title': 'Quiz 3'
}])
quiz = Quiz(3, course_id)
views.db.session.add(quiz)
views.db.session.commit()
job = self.queue.enqueue_call(func=refresh_background,
args=(course_id, ))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
job_result = job.result
meta_keys = ['status', 'status_msg', 'percent', 'error']
self.assertTrue(all(key in job_result for key in meta_keys))
self.assertEqual(job_result['status'], 'complete')
self.assertEqual(job_result['status_msg'],
'Complete. No quizzes required updates.')
self.assertEqual(job_result['percent'], 100)
self.assertFalse(job_result['error'])
def test_refresh_background_update_error(self, m):
from views import refresh_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
m.register_uri('GET',
'/api/v1/courses/{}'.format(course_id),
json={
'id': course_id,
'name': 'Example Course'
})
m.register_uri('GET',
'/api/v1/courses/1/quizzes',
json=[{
'id': 1,
'title': 'Quiz 1',
'time_limit': 10
}])
m.register_uri('POST',
'/api/v1/courses/1/quizzes/1/extensions',
status_code=404)
m.register_uri('GET',
'/api/v1/courses/1/users/12345',
json={
'id': 12345,
'sortable_name': 'John Smith',
'enrollments': [{
'type': 'StudentEnrollment'
}]
})
course = Course(course_id, course_name='Example Course')
views.db.session.add(course)
user = User(12345, sortable_name="John Smith")
views.db.session.add(user)
views.db.session.commit()
ext = Extension(course.id, user.id)
views.db.session.add(ext)
views.db.session.commit()
job = self.queue.enqueue_call(func=refresh_background,
args=(course_id, ))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
job_result = job.result
meta_keys = ['status', 'status_msg', 'percent', 'error']
self.assertTrue(all(key in job_result for key in meta_keys))
self.assertEqual(job_result['status'], 'failed')
self.assertTrue(job_result['error'])
self.assertEqual(
job_result['status_msg'],
('Some quizzes couldn\'t be updated. Error creating extension '
'for quiz #1. Canvas status code: 404'))
def test_refresh_background_inactive_user(self, m):
from views import refresh_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
user_id = 9001
m.register_uri('GET',
'/api/v1/courses/{}'.format(course_id),
json={
'id': course_id,
'name': 'Example Course'
})
m.register_uri('GET',
'/api/v1/courses/{}/quizzes'.format(course_id),
json=[{
'id': 1,
'title': 'Quiz 1',
'time_limit': 10
}, {
'id': 2,
'title': 'Quiz 2',
'time_limit': 30
}])
m.register_uri(
'POST',
'/api/v1/courses/{}/quizzes/1/extensions'.format(course_id),
status_code=200)
m.register_uri(
'POST',
'/api/v1/courses/{}/quizzes/2/extensions'.format(course_id),
status_code=200)
m.register_uri('GET',
'/api/v1/courses/{}/users/{}'.format(
course_id, user_id),
status_code=404)
course = Course(course_id, course_name='Example Course')
views.db.session.add(course)
user = User(user_id, sortable_name="Missing User")
views.db.session.add(user)
views.db.session.commit()
ext = Extension(course.id, user.id)
views.db.session.add(ext)
views.db.session.commit()
# Check that the extension is active first
self.assertTrue(ext.active)
ext_id = ext.id
job = self.queue.enqueue_call(func=refresh_background,
args=(course_id, ))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
# Ensure extension has been marked as inactive.
extension = views.db.session.query(Extension).filter_by(
id=ext_id).first()
self.assertFalse(extension.active)
def test_refresh_background_update_success(self, m):
from views import refresh_background
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
course_id = 1
m.register_uri('GET',
'/api/v1/courses/{}'.format(course_id),
json={
'id': course_id,
'name': 'Example Course'
})
m.register_uri('GET',
'/api/v1/courses/1/quizzes',
json=[{
'id': 1,
'title': 'Quiz 1',
'time_limit': 10
}, {
'id': 2,
'title': 'Quiz 2',
'time_limit': 30
}])
m.register_uri('POST',
'/api/v1/courses/1/quizzes/1/extensions',
status_code=200)
m.register_uri('POST',
'/api/v1/courses/1/quizzes/2/extensions',
status_code=200)
m.register_uri('GET',
'/api/v1/courses/1/users/12345',
json={
'id': 12345,
'sortable_name': 'John Smith',
'enrollments': [{
'type': 'StudentEnrollment'
}]
})
course = Course(course_id, course_name='Example Course')
views.db.session.add(course)
user = User(12345, sortable_name="John Smith")
views.db.session.add(user)
views.db.session.commit()
ext = Extension(course.id, user.id)
views.db.session.add(ext)
# Add an inactive extension to be ignored.
ext_inactive = Extension(course.id, user.id)
ext_inactive.active = False
views.db.session.add(ext_inactive)
views.db.session.commit()
job = self.queue.enqueue_call(func=refresh_background,
args=(course_id, ))
self.worker.work(burst=True)
self.assertTrue(job.is_finished)
job_result = job.result
meta_keys = ['status', 'status_msg', 'percent', 'error']
self.assertTrue(all(key in job_result for key in meta_keys))
self.assertEqual(job_result['status'], 'complete')
self.assertFalse(job_result['error'])
self.assertEqual(job_result['status_msg'],
'2 quizzes have been updated.')
self.assertEqual(job_result['percent'], 100)
def test_missing_quizzes_check_no_course(self, m):
course_id = 1
response = self.client.get('/missing_quizzes/{}/'.format(course_id))
self.assert_200(response)
self.assertEqual(response.data, 'false')
def test_missing_quizzes_check_no_extensions(self, m):
course_id = 1
course = Course(canvas_id=course_id, course_name='test')
views.db.session.add(course)
views.db.session.commit()
response = self.client.get('/missing_quizzes/{}/'.format(course_id))
self.assert_200(response)
self.assertEqual(response.data, 'false')
def test_missing_quizzes_check_true(self, m):
m.register_uri('GET',
'/api/v1/courses/1/quizzes',
json=[{
'id': 1,
'title': 'Quiz 1'
}])
course_id = 1
course = Course(canvas_id=course_id, course_name='test')
views.db.session.add(course)
views.db.session.commit()
extension = Extension(course_id=course.id, user_id=5, percent=200)
views.db.session.add(extension)
views.db.session.commit()
response = self.client.get('/missing_quizzes/{}/'.format(course_id))
self.assert_200(response)
self.assertEqual(response.data, 'true')
def test_missing_quizzes_check_false(self, m):
m.register_uri('GET',
'/api/v1/courses/1/quizzes',
json=[{
'id': 1,
'title': 'Quiz 1'
}])
course_id = 1
course = Course(canvas_id=course_id, course_name='test')
views.db.session.add(course)
quiz = Quiz(canvas_id=1, course_id=course.id)
views.db.session.add(quiz)
views.db.session.commit()
extension = Extension(course_id=course.id, user_id=5, percent=200)
views.db.session.add(extension)
views.db.session.commit()
response = self.client.get('/missing_quizzes/{}/'.format(course_id))
self.assert_200(response)
self.assertEqual(response.data, 'false')
def test_filter_no_students_found(self, m):
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
m.register_uri('GET', '/api/v1/courses/1/search_users', json=[])
course_id = 1
response = self.client.get('/filter/{}/'.format(course_id))
self.assert_200(response)
self.assert_template_used('user_list.html')
self.assertEqual(len(self.get_context_variable('users')), 0)
self.assertEqual(self.get_context_variable('max_pages'), 1)
def test_filter(self, m):
with self.client.session_transaction() as sess:
sess['canvas_user_id'] = 1234
sess['lti_logged_in'] = True
sess['is_admin'] = True
m.register_uri(
'GET',
'/api/v1/courses/1/search_users',
json=[{
'id': 1,
'name': 'John Smith'
}, {
'id': 2,
'name': 'Jane Doe'
}],
headers={
"Link":
"<http://example.com/api/v1/courses/1/search_users?page=99>; rel=\"last\""
})
course_id = 1
response = self.client.get('/filter/{}/'.format(course_id))
self.assert_200(response)
self.assert_template_used('user_list.html')
self.assertEqual(len(self.get_context_variable('users')), 2)
self.assertEqual(self.get_context_variable('max_pages'), 99)
def test_lti_tool_not_admin_or_instructor(self, m):
user_id = 42
response = self.client.post('/launch',
data={
'custom_canvas_course_id': 'test',
'custom_canvas_user_id': user_id,
'custom_canvas_api_domain':
config.TESTING_API_URL,
'ext_roles': []
})
self.assert_200(response)
self.assert_template_used('error.html')
self.assertEqual(self.get_context_variable('message'),
'Must be an Administrator or Instructor')
def test_lti_tool(self, m):
user_id = 42
with self.client as c:
response = c.post('/launch',
data={
'custom_canvas_course_id': 'test',
'custom_canvas_user_id': user_id,
'custom_canvas_api_domain':
config.TESTING_API_URL,
'ext_roles': 'Administrator'
})
self.assert200(response)
self.assertTrue(session.get('is_admin', False))
def execute_all_jobs(worker_queue, redis_con):
worker = SimpleWorker([worker_queue], connection=redis_con)
worker.work(burst=True)
def test_monitor_job_with_retry(client):
"""Test monitoring a job for a task that fails"""
with client.application.app_context():
app = client.application
app.redis.flushall()
task, job, execution = JobExecutionFixture.new_defaults()
job.metadata["retries"] = 3
job.metadata["retry_count"] = 0
job.save()
job_id = str(job.job_id)
exec_mock = MagicMock()
exec_mock.get_result.return_value = MagicMock(
exit_code=1, log="".encode("utf-8"), error="error".encode("utf-8"))
client.application.executor = exec_mock
queue = Queue("monitor",
is_async=False,
connection=client.application.redis)
result = queue.enqueue(job_mod.monitor_job, task.task_id, job_id,
execution.execution_id)
worker = SimpleWorker([queue], connection=queue.connection)
worker.work(burst=True)
task.reload()
expect(task.jobs).to_length(1)
job = task.jobs[0]
expect(job.executions).to_length(2)
execution = job.executions[0]
expect(execution.image).to_equal("image")
expect(execution.command).to_equal("command")
hash_key = f"rq:job:{result.id}"
res = app.redis.exists(hash_key)
expect(res).to_be_true()
res = app.redis.hget(hash_key, "status")
expect(res).to_equal("finished")
res = app.redis.hexists(hash_key, "data")
expect(res).to_be_true()
res = app.redis.zrange(b"rq:scheduler:scheduled_jobs", 0, -1)
expect(res).to_length(1)
time = datetime.now() + timedelta(seconds=2)
res = app.redis.zscore("rq:scheduler:scheduled_jobs", res[0])
expect(int(res)).to_be_greater_than(int(time.timestamp()) - 2)
expect(int(res)).to_be_lesser_than(int(time.timestamp()) + 2)
new_job = app.redis.zrange("rq:scheduler:scheduled_jobs", 0,
0)[0].decode("utf-8")
next_job_id = f"rq:job:{new_job}"
res = app.redis.exists(next_job_id)
expect(res).to_be_true()
res = app.redis.hexists(next_job_id, "data")
expect(res).to_be_true()
res = app.redis.hget(next_job_id, "origin")
expect(res).to_equal("jobs")
res = app.redis.hget(next_job_id, "description")
job.reload()
expect(res).to_equal(
(f"fastlane.worker.job.run_job('{task.task_id}', '{job_id}', "
f"'{job.executions[-1].execution_id}', 'image', 'command')"))
task.reload()
expect(task.jobs[0].executions[0].status).to_equal(
JobExecution.Status.failed)
def test_run_job1(client):
"""Test running a new job for a task"""
with client.application.app_context():
app = client.application
app.redis.flushall()
task, job, execution = JobExecutionFixture.new_defaults()
exec_mock = MagicMock()
exec_mock.validate_max_running_executions.return_value = True
client.application.executor = exec_mock
queue = Queue("jobs",
is_async=False,
connection=client.application.redis)
result = queue.enqueue(
job_mod.run_job,
task.task_id,
job.job_id,
execution.execution_id,
"image",
"command",
)
worker = SimpleWorker([queue], connection=queue.connection)
worker.work(burst=True)
task.reload()
expect(task.jobs).to_length(1)
job = task.jobs[0]
expect(job.executions).to_length(1)
execution = job.executions[0]
expect(execution.image).to_equal("image")
expect(execution.command).to_equal("command")
hash_key = "rq:scheduler:scheduled_jobs"
res = app.redis.exists(hash_key)
expect(res).to_be_true()
res = app.redis.zrange(hash_key, 0, -1)
expect(res).to_length(1)
monitored_key = res[0].decode("utf-8")
hash_key = f"rq:job:{result.id}"
res = app.redis.exists(hash_key)
expect(res).to_be_true()
res = app.redis.hget(hash_key, "status")
expect(res).to_equal("finished")
res = app.redis.hexists(hash_key, "data")
expect(res).to_be_true()
next_job_id = f"rq:job:{monitored_key}"
res = app.redis.exists(next_job_id)
expect(res).to_be_true()
res = app.redis.hexists(next_job_id, "data")
expect(res).to_be_true()
res = app.redis.hget(next_job_id, "origin")
expect(res).to_equal("monitor")
res = app.redis.hget(next_job_id, "description")
expect(res).to_equal(
f"fastlane.worker.job.monitor_job('{task.task_id}', "
f"'{job.job_id}', '{execution.execution_id}')")
task.reload()
expect(task.jobs[0].executions[0].status).to_equal(
JobExecution.Status.running)
def run_worker():
redis_url = app.config["REDIS_URL"]
redis_connection = redis.from_url(redis_url)
with Connection(redis_connection):
worker = SimpleWorker(app.config["QUEUES"])
worker.work()
def execute_all_jobs(worker_queue, redis_con):
worker = SimpleWorker([worker_queue], connection=redis_con)
worker.work(burst=True)
def workerbee(redisServer, task):
print("the TASK is %s" % task)
queue = Queue(connection=Redis.from_url(redisServer))
worker = SimpleWorker([queue], connection=queue.connection)
worker.work()
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import redis
from rq import Connection
from rq import Queue
from rq import SimpleWorker
from code_coverage_backend import secrets
conn = redis.from_url(secrets.REDIS_URL)
def exc_handler(job, *exc_info):
job.cleanup(ttl=3600)
if __name__ == '__main__':
with Connection(conn):
worker = SimpleWorker(map(Queue, ['default']), exception_handlers=[])
worker.push_exc_handler(exc_handler)
worker.push_exc_handler(worker.move_to_failed_queue)
worker.work()
from app.redis import conn
from okr.scrapers import scheduler as scheduler_okr
from bot_seo import scheduler as scheduler_bot_seo
scheduler_modules = [
scheduler_okr,
scheduler_bot_seo,
]
# Set up schedulers
for scheduler_module in scheduler_modules:
scheduler_module.setup()
if not settings.DEBUG:
scheduler_module.add_jobs()
# Start rq connection
listen = ["high", "default", "low"]
if __name__ == "__main__":
with Connection(conn):
worker = SimpleWorker(map(Queue, listen))
worker.work() # Blocking
# Cleanup
for scheduler_module in scheduler_modules:
scheduler_module.scheduler.shutdown()
if getattr(scheduler_module, "executors", None):
for executor in scheduler_module.executors.values():
executor.shutdown()
