def test_process_other_queue(self):
named_queue_task('test3')
tasks.run_next_task(queue='other_named_queue')
self.assertNotIn('test3',
completed_named_queue_tasks,
msg='Task should be ignored')
tasks.run_next_task()
def test_issue_method(self):
acn = django_vox.models.Notification.objects.get_by_natural_key(
"tests", "article", "created"
)
article = models.Article.objects.get(pk="too_many")
user = auth_models.User.objects.get(pk=1)
# first test the background issuing
django_vox.models.get_issue_function.cache_clear()
with override_settings(
DJANGO_VOX_ISSUE_METHOD="django_vox.extra.background_tasks.issue"
):
acn.issue(article, actor=user, target=article)
from background_task.models import Task
all_tasks = Task.objects.all()
assert len(all_tasks) == 1
task = all_tasks[0]
assert task.queue == "django-vox"
assert task.task_name == "django_vox.extra.background_tasks.delayed_issue"
# now lets try to manually run the task
from background_task.tasks import tasks
assert 0 == len(mail.outbox)
tasks.run_next_task("django-vox")
assert 3 == len(mail.outbox)
django_vox.models.get_issue_function.cache_clear()
with override_settings(DJANGO_VOX_ISSUE_METHOD="invalid_method!!!"):
with self.assertRaises(ImproperlyConfigured):
acn.issue(article, actor=user, target=article)
def test_task_with_dictionary_in_args(self):
self.assertEqual(Task.objects.count(), 0)
d = {22222: 2, 11111: 1}
self.task(d)
self.assertEqual(Task.objects.count(), 1)
tasks.run_next_task()
self.assertEqual(Task.objects.count(), 0)
def test_repetition_in_future(self):
repeat_until = timezone.now() + timedelta(weeks=1)
old_task = self.my_task(
'test-repetition',
repeat=Task.HOURLY,
repeat_until=repeat_until,
verbose_name="Test repetition in future",
)
old_task.run_at = timezone.now() - timedelta(
weeks=1) # task is one week old
old_task.save()
tasks.run_next_task()
time.sleep(0.5)
self.assertEqual(Task.objects.filter(repeat=Task.HOURLY).count(), 1)
new_task = Task.objects.get(repeat=Task.HOURLY)
self.assertNotEqual(new_task.id, old_task.id)
# new task skipped exactly one week of downtime in the past, keeps period
self.assertEqual((new_task.run_at - old_task.run_at),
timedelta(weeks=1, hours=1))
# new task will be executed in the future
self.assertTrue(new_task.run_at > timezone.now())
# new task will be executed in less than one hour
self.assertTrue(
(new_task.run_at - timezone.now()) <= timedelta(hours=1))
def test_task_with_dictionary_in_args(self):
self.assertEqual(Task.objects.count(), 0)
d = {random.randint(1,1000):random.randint(1,1000) for _ in range(random.randint(1,10))}
self.task(d)
self.assertEqual(Task.objects.count(), 1, 'Task dont created')
tasks.run_next_task()
self.assertEqual(Task.objects.count(), 0, 'Task dont started')
def test_transfer_delivery_no_recipient(self, mock_azure_transfer):
mock_azure_transfer.return_value.email_sender.return_value.email_text = "emailtext1"
mock_azure_transfer.return_value.email_recipient.return_value.email_text = "emailtext2"
TransferFunctions.transfer_delivery(delivery_id='123', transfer_project=False, add_download_users=False,
change_owner=False, email_sender=False, email_recipient=False)
tasks.run_next_task()
expected_calls = [
call.ensure_transferring_state(),
call.mark_complete()
]
mock_azure_transfer.return_value.assert_has_calls(expected_calls)
def test_run_next_task_several_tasks_scheduled(self):
self.set_fields(one='1')
self.set_fields(two='2')
self.set_fields(three='3')
for i in range(3):
self.failUnless(tasks.run_next_task())
self.failIf(tasks.run_next_task()) # everything should have been run
for field, value in [('one', '1'), ('two', '2'), ('three', '3')]:
self.failUnless(hasattr(self, field))
self.assertEqual(value, getattr(self, field))
def test_run_next_task_several_tasks_scheduled(self):
self.set_fields(one='1')
self.set_fields(two='2')
self.set_fields(three='3')
for i in range(3):
self.failUnless(tasks.run_next_task())
self.failIf(tasks.run_next_task()) # everything should have been run
for field, value in [('one', '1'), ('two', '2'), ('three', '3')]:
self.failUnless(hasattr(self, field))
self.assertEqual(value, getattr(self, field))
def test_failed_at_set_after_MAX_ATTEMPTS(self):
@tasks.background(name='test_failed_at_set_after_MAX_ATTEMPTS')
def failed_at_set_after_MAX_ATTEMPTS():
raise RuntimeError('failed')
failed_at_set_after_MAX_ATTEMPTS()
available = Task.objects.find_available()
self.assertEqual(1, available.count())
task = available[0]
self.failUnless(task.failed_at is None)
task.attempts = settings.MAX_ATTEMPTS
task.save()
# task should be scheduled to run now
# but will be marked as failed straight away
self.failUnless(tasks.run_next_task())
available = Task.objects.find_available()
self.assertEqual(0, available.count())
all_tasks = Task.objects.all()
self.assertEqual(0, all_tasks.count())
self.assertEqual(1, CompletedTask.objects.count())
completed_task = CompletedTask.objects.all()[0]
self.failIf(completed_task.failed_at is None)
def test_failed_at_set_after_MAX_ATTEMPTS(self):
@tasks.background(name='test_failed_at_set_after_MAX_ATTEMPTS')
def failed_at_set_after_MAX_ATTEMPTS():
raise RuntimeError('failed')
failed_at_set_after_MAX_ATTEMPTS()
available = Task.objects.find_available()
self.assertEqual(1, available.count())
task = available[0]
self.failUnless(task.failed_at is None)
task.attempts = settings.MAX_ATTEMPTS
task.save()
# task should be scheduled to run now
# but will be marked as failed straight away
self.failUnless(tasks.run_next_task())
available = Task.objects.find_available()
self.assertEqual(0, available.count())
all_tasks = Task.objects.all()
self.assertEqual(0, all_tasks.count())
self.assertEqual(1, CompletedTask.objects.count())
completed_task = CompletedTask.objects.all()[0]
self.failIf(completed_task.failed_at is None)
def test_transfer_delivery(self, mock_azure_transfer):
mock_azure_transfer.return_value.email_sender.return_value.email_text = "emailtext1"
mock_azure_transfer.return_value.email_recipient.return_value.email_text = "emailtext2"
TransferFunctions.transfer_delivery(delivery_id='123')
tasks.run_next_task()
expected_calls = [
call.ensure_transferring_state(),
call.record_object_manifest(),
call.transfer_project(),
call.give_download_users_permissions(),
call.update_owner_permissions(),
call.email_sender(),
call.email_recipient(),
call.mark_complete()
]
mock_azure_transfer.return_value.assert_has_calls(expected_calls)
def test_run_next_task_one_task_scheduled(self):
self.set_fields(worked=True)
self.failIf(hasattr(self, 'worked'))
self.failUnless(tasks.run_next_task())
self.failUnless(hasattr(self, 'worked'))
self.failUnless(self.worked)
def test_run_next_task_one_task_scheduled(self):
self.set_fields(worked=True)
self.failIf(hasattr(self, 'worked'))
self.failUnless(tasks.run_next_task())
self.failUnless(hasattr(self, 'worked'))
self.failUnless(self.worked)
def mocked_run_next_task(queue=None):
"""
We mock tasks.mocked_run_next_task to give other threads some time to update the database.
Otherwise we run into a locked database.
"""
val = tasks.run_next_task(queue)
if app_settings.BACKGROUND_TASK_RUN_ASYNC:
time.sleep(1)
return val
def mocked_run_next_task(queue=None):
"""
We mock tasks.mocked_run_next_task to give other threads some time to update the database.
Otherwise we run into a locked database.
"""
val = tasks.run_next_task(queue)
if app_settings.BACKGROUND_TASK_RUN_ASYNC:
time.sleep(1)
return val
def test_max_attempts_one(self):
with self.settings(MAX_ATTEMPTS=1):
self.assertEqual(settings.MAX_ATTEMPTS, 1)
self.assertEqual(Task.objects.count(), 2)
tasks.run_next_task()
self.assertEqual(Task.objects.count(), 1)
self.assertEqual(Task.objects.all()[0].id, self.task2_id)
self.assertEqual(CompletedTask.objects.count(), 1)
completed_task = CompletedTask.objects.all()[0]
self.assertEqual(completed_task.attempts, 1)
self.assertEqual(completed_task.task_name, self.task1.task_name)
self.assertEqual(completed_task.task_params, self.task1.task_params)
self.assertIsNotNone(completed_task.last_error)
self.assertIsNotNone(completed_task.failed_at)
tasks.run_next_task()
self.assertEqual(Task.objects.count(), 0)
self.assertEqual(CompletedTask.objects.count(), 2)
def test_max_attempts_one(self):
with self.settings(MAX_ATTEMPTS=1):
self.assertEqual(settings.MAX_ATTEMPTS, 1)
self.assertEqual(Task.objects.count(), 2)
tasks.run_next_task()
self.assertEqual(Task.objects.count(), 1)
self.assertEqual(Task.objects.all()[0].id, self.task2_id)
self.assertEqual(CompletedTask.objects.count(), 1)
completed_task = CompletedTask.objects.all()[0]
self.assertEqual(completed_task.attempts, 1)
self.assertEqual(completed_task.task_name, self.task1.task_name)
self.assertEqual(completed_task.task_params,
self.task1.task_params)
self.assertIsNotNone(completed_task.last_error)
self.assertIsNotNone(completed_task.failed_at)
tasks.run_next_task()
self.assertEqual(Task.objects.count(), 0)
self.assertEqual(CompletedTask.objects.count(), 2)
def test_repetition_in_future(self):
repeat_until = timezone.now() + timedelta(weeks=1)
old_task = self.my_task(
'test-repetition',
repeat=Task.HOURLY,
repeat_until=repeat_until,
verbose_name="Test repetition in future",
)
old_task.run_at = timezone.now() - timedelta(weeks=1) # task is one week old
old_task.save()
tasks.run_next_task()
time.sleep(0.5)
self.assertEqual(Task.objects.filter(repeat=Task.HOURLY).count(), 1)
new_task = Task.objects.get(repeat=Task.HOURLY)
self.assertNotEqual(new_task.id, old_task.id)
# new task skipped exactly one week of downtime in the past, keeps period
self.assertEqual((new_task.run_at - old_task.run_at), timedelta(weeks=1, hours=1))
# new task will be executed in the future
self.assertTrue(new_task.run_at > timezone.now())
# new task will be executed in less than one hour
self.assertTrue((new_task.run_at - timezone.now()) <= timedelta(hours=1))
def test_repeat(self):
repeat_until = timezone.now() + timedelta(weeks=1)
old_task = self.my_task(
'test-repeat',
foo='bar',
repeat=Task.HOURLY,
repeat_until=repeat_until,
verbose_name="Test repeat",
)
self.assertEqual(old_task.repeat, Task.HOURLY)
self.assertEqual(old_task.repeat_until, repeat_until)
tasks.run_next_task()
time.sleep(0.5)
self.assertEqual(Task.objects.filter(repeat=Task.HOURLY).count(), 1)
new_task = Task.objects.get(repeat=Task.HOURLY)
self.assertNotEqual(new_task.id, old_task.id)
self.assertEqual(new_task.task_name, old_task.task_name)
self.assertEqual(new_task.params(), old_task.params())
self.assertEqual(new_task.task_hash, old_task.task_hash)
self.assertEqual(new_task.verbose_name, old_task.verbose_name)
self.assertEqual((new_task.run_at - old_task.run_at), timedelta(hours=1))
self.assertEqual(new_task.repeat_until, old_task.repeat_until)
def test_run_next_task_does_not_run_locked(self):
self.set_fields(locked=True)
self.failIf(hasattr(self, 'locked'))
all_tasks = Task.objects.all()
self.assertEqual(1, all_tasks.count())
original_task = all_tasks[0]
original_task.lock('lockname')
self.failIf(tasks.run_next_task())
self.failIf(hasattr(self, 'locked'))
all_tasks = Task.objects.all()
self.assertEqual(1, all_tasks.count())
def test_repeat(self):
repeat_until = timezone.now() + timedelta(weeks=1)
old_task = self.my_task(
'test-repeat',
foo='bar',
repeat=Task.HOURLY,
repeat_until=repeat_until,
verbose_name="Test repeat",
)
self.assertEqual(old_task.repeat, Task.HOURLY)
self.assertEqual(old_task.repeat_until, repeat_until)
tasks.run_next_task()
time.sleep(0.5)
self.assertEqual(Task.objects.filter(repeat=Task.HOURLY).count(), 1)
new_task = Task.objects.get(repeat=Task.HOURLY)
self.assertNotEqual(new_task.id, old_task.id)
self.assertEqual(new_task.task_name, old_task.task_name)
self.assertEqual(new_task.params(), old_task.params())
self.assertEqual(new_task.task_hash, old_task.task_hash)
self.assertEqual(new_task.verbose_name, old_task.verbose_name)
self.assertEqual((new_task.run_at - old_task.run_at), timedelta(hours=1))
self.assertEqual(new_task.repeat_until, old_task.repeat_until)
def test_run_next_task_does_not_run_locked(self):
self.set_fields(locked=True)
self.failIf(hasattr(self, 'locked'))
all_tasks = Task.objects.all()
self.assertEqual(1, all_tasks.count())
original_task = all_tasks[0]
original_task.lock('lockname')
self.failIf(tasks.run_next_task())
self.failIf(hasattr(self, 'locked'))
all_tasks = Task.objects.all()
self.assertEqual(1, all_tasks.count())
def test_run_next_task_unlocks_after_MAX_RUN_TIME(self):
self.set_fields(lock_overridden=True)
all_tasks = Task.objects.all()
self.assertEqual(1, all_tasks.count())
original_task = all_tasks[0]
locked_task = original_task.lock('lockname')
self.failIf(tasks.run_next_task())
self.failIf(hasattr(self, 'lock_overridden'))
# put lot time into past
expire_by = timedelta(seconds=(settings.MAX_RUN_TIME + 2))
locked_task.locked_at = locked_task.locked_at - expire_by
locked_task.save()
# so now we should be able to override the lock
# and run the task
self.failUnless(tasks.run_next_task())
self.assertEqual(0, Task.objects.count())
self.failUnless(hasattr(self, 'lock_overridden'))
self.failUnless(self.lock_overridden)
def test_run_next_task_unlocks_after_MAX_RUN_TIME(self):
self.set_fields(lock_overridden=True)
all_tasks = Task.objects.all()
self.assertEqual(1, all_tasks.count())
original_task = all_tasks[0]
locked_task = original_task.lock('lockname')
self.failIf(tasks.run_next_task())
self.failIf(hasattr(self, 'lock_overridden'))
# put lot time into past
expire_by = timedelta(seconds=(settings.MAX_RUN_TIME + 2))
locked_task.locked_at = locked_task.locked_at - expire_by
locked_task.save()
# so now we should be able to override the lock
# and run the task
self.failUnless(tasks.run_next_task())
self.assertEqual(0, Task.objects.count())
self.failUnless(hasattr(self, 'lock_overridden'))
self.failUnless(self.lock_overridden)
def handle(self, *args, **options):
log_level = options.pop('log_level', None)
log_file = options.pop('log_file', None)
log_std = options.pop('log_std', False)
duration = options.pop('duration', 0)
sleep = options.pop('sleep', 5.0)
self._configure_logging(log_level, log_file, log_std)
autodiscover()
start_time = time.time()
while (duration <= 0) or (time.time() - start_time) <= duration:
if not tasks.run_next_task():
logging.debug('waiting for tasks')
time.sleep(sleep)
def test_run_next_task_error_handling(self):
self.throws_error()
all_tasks = Task.objects.all()
self.assertEqual(1, all_tasks.count())
original_task = all_tasks[0]
# should run, but trigger error
self.failUnless(tasks.run_next_task())
all_tasks = Task.objects.all()
self.assertEqual(1, all_tasks.count())
failed_task = all_tasks[0]
# should have an error recorded
self.failIfEqual('', failed_task.last_error)
self.failUnless(failed_task.failed_at is None)
self.assertEqual(1, failed_task.attempts)
# should have been rescheduled for the future
# and no longer locked
self.failUnless(failed_task.run_at > original_task.run_at)
self.failUnless(failed_task.locked_by is None)
self.failUnless(failed_task.locked_at is None)
def test_run_next_task_error_handling(self):
self.throws_error()
all_tasks = Task.objects.all()
self.assertEqual(1, all_tasks.count())
original_task = all_tasks[0]
# should run, but trigger error
self.failUnless(tasks.run_next_task())
all_tasks = Task.objects.all()
self.assertEqual(1, all_tasks.count())
failed_task = all_tasks[0]
# should have an error recorded
self.failIfEqual('', failed_task.last_error)
self.failUnless(failed_task.failed_at is None)
self.assertEqual(1, failed_task.attempts)
# should have been rescheduled for the future
# and no longer locked
self.failUnless(failed_task.run_at > original_task.run_at)
self.failUnless(failed_task.locked_by is None)
self.failUnless(failed_task.locked_at is None)
def mocked_run_next_task(queue=None):
"""
We mock tasks.mocked_run_next_task to give other threads some time to update the database.
Otherwise we run into a locked database.
"""
return tasks.run_next_task(queue)
def test_process_all_tasks(self):
named_queue_task('test2')
tasks.run_next_task()
self.assertIn('test2',
completed_named_queue_tasks,
msg='Task should be processed')
def test_process_queue(self):
named_queue_task('test1')
tasks.run_next_task(queue='named_queue')
self.assertIn('test1',
completed_named_queue_tasks,
msg='Task should be processed')
def test_lock_contested(self): # locking should actually look at db, not object # in memory task = Task.objects.new_task('mytask') task.save() self.failIf(task.lock('mylock') is None)
self.failUnless(task.lock('otherlock') is None)
def test_lock_expired(self): settings.MAX_RUN_TIME = 60 task = Task.objects.new_task('mytask') task.save() locked_task = task.lock('mylock')
# force expire the lock expire_by = timedelta(seconds=(settings.MAX_RUN_TIME + 2)) locked_task.locked_at = locked_task.locked_at - expire_by locked_task.save()
# now try to get the lock again self.failIf(task.lock('otherlock') is None)
def test__unicode__(self): task = Task.objects.new_task('mytask') self.assertEqual(u'Task(mytask)', unicode(task))
class TestTasks(TransactionTestCase):
def setUp(self): super(TestTasks, self).setUp()
settings.MAX_RUN_TIME = 60 settings.MAX_ATTEMPTS = 25
@tasks.background(name='set_fields') def set_fields(**fields): for key, value in fields.items(): setattr(self, key, value)
@tasks.background(name='throws_error') def throws_error(): raise RuntimeError("an error")
self.set_fields = set_fields self.throws_error = throws_error
def test_run_next_task_nothing_scheduled(self): self.failIf(tasks.run_next_task())
def test_run_next_task_one_task_scheduled(self): self.set_fields(worked=True) self.failIf(hasattr(self, 'worked'))
self.failUnless(tasks.run_next_task())
self.failUnless(hasattr(self, 'worked')) self.failUnless(self.worked)
def test_run_next_task_several_tasks_scheduled(self): self.set_fields(one='1') self.set_fields(two='2') self.set_fields(three='3')
for i in range(3): self.failUnless(tasks.run_next_task())
self.failIf(tasks.run_next_task()) # everything should have been run
for field, value in [('one', '1'), ('two', '2'), ('three', '3')]: self.failUnless(hasattr(self, field)) self.assertEqual(value, getattr(self, field))
def test_run_next_task_error_handling(self): self.throws_error()
all_tasks = Task.objects.all() self.assertEqual(1, all_tasks.count()) original_task = all_tasks[0]
# should run, but trigger error self.failUnless(tasks.run_next_task())
all_tasks = Task.objects.all() self.assertEqual(1, all_tasks.count())
failed_task = all_tasks[0] # should have an error recorded self.failIfEqual('', failed_task.last_error) self.failUnless(failed_task.failed_at is None) self.assertEqual(1, failed_task.attempts)
# should have been rescheduled for the future # and no longer locked self.failUnless(failed_task.run_at > original_task.run_at) self.failUnless(failed_task.locked_by is None) self.failUnless(failed_task.locked_at is None)
def test_run_next_task_does_not_run_locked(self): self.set_fields(locked=True) self.failIf(hasattr(self, 'locked'))
all_tasks = Task.objects.all() self.assertEqual(1, all_tasks.count()) original_task = all_tasks[0] original_task.lock('lockname')
self.failIf(tasks.run_next_task())
self.failIf(hasattr(self, 'locked')) all_tasks = Task.objects.all() self.assertEqual(1, all_tasks.count())
def test_max_attempts_two(self):
with self.settings(MAX_ATTEMPTS=2):
self.assertEqual(settings.MAX_ATTEMPTS, 2)
tasks.run_next_task()
self.assertEqual(Task.objects.count(), 2)
self.assertEqual(CompletedTask.objects.count(), 0)
def test_run_next_task_nothing_scheduled(self): self.failIf(tasks.run_next_task()) def test_run_next_task_one_task_scheduled(self): self.set_fields(worked=True) self.failIf(hasattr(self, 'worked'))
def test_publish(self):
"""
Test publishing project
"""
# Get the project ready to publish
self.test_approve_publish()
self.client.login(username='******', password='******')
project = ActiveProject.objects.get(
title='MIT-BIH Arrhythmia Database')
project_slug = project.slug
custom_slug = 'mitbih'
# Try to publish with an already taken slug
# (note that if the project is a new version,
# publish_submission ignores the slug parameter)
if not project.is_new_version:
taken_slug = PublishedProject.objects.all().first().slug
response = self.client.post(reverse('publish_submission',
args=(project.slug, )),
data={
'slug': taken_slug,
'doi': False,
'make_zip': 1
})
self.assertTrue(
bool(ActiveProject.objects.filter(slug=project_slug)))
# Publish with a valid custom slug
response = self.client.post(reverse('publish_submission',
args=(project.slug, )),
data={
'slug': custom_slug,
'doi': False,
'make_zip': 1
})
# Run background tasks
self.assertTrue(bool(tasks.run_next_task()))
self.assertTrue(bool(
PublishedProject.objects.filter(slug=custom_slug)))
self.assertFalse(
bool(PublishedProject.objects.filter(slug=project_slug)))
self.assertFalse(bool(ActiveProject.objects.filter(slug=project_slug)))
project = PublishedProject.objects.get(slug=custom_slug,
version=project.version)
# Access the published project's page and its (open) files
response = self.client.get(
reverse('published_project', args=(project.slug, project.version)))
self.assertEqual(response.status_code, 200)
response = self.client.get(
reverse('serve_published_project_file',
args=(project.slug, project.version,
'subject-100/100.atr')))
self.assertEqual(response.status_code, 200)
response = self.client.get(
reverse('serve_published_project_zip',
args=(project.slug, project.version)))
self.assertEqual(response.status_code, 200)
# Access the submission log as the author
self.client.login(username='******', password='******')
response = self.client.get(
reverse('published_submission_history',
args=(
project.slug,
project.version,
)))
self.assertEqual(response.status_code, 200)
def test_run_next_task_nothing_scheduled(self):
self.failIf(tasks.run_next_task())
def test_run_next_task_nothing_scheduled(self):
self.failIf(tasks.run_next_task())
def test_max_attempts_two(self):
with self.settings(MAX_ATTEMPTS=2):
self.assertEqual(settings.MAX_ATTEMPTS, 2)
tasks.run_next_task()
self.assertEqual(Task.objects.count(), 2)
self.assertEqual(CompletedTask.objects.count(), 0)