def test_missing_genome_section(self):
expected_counts = {
'batches': 1,
'results_per_sample': [
{
'batch': 'batch1',
'sample': 'NA12891',
'count': 1963,
},
{
'batch': 'batch1',
'sample': 'NA12892',
'count': 1963,
},
{
'batch': 'batch1',
'sample': 'NA12878',
'count': 1963,
}
],
'samples': 3,
'samples_per_batch': [(1, 3)],
}
expected_counts['results'] = \
sum([x['count'] for x in expected_counts['results_per_sample']])
# Immediately validates and creates a sample.
management.call_command('samples', 'queue')
# Synchronously work on queue.
worker1 = get_worker('variants')
worker2 = get_worker('default')
# Work on variants.
worker1.work(burst=True)
# Work on effects.
worker2.work(burst=True)
# Since the MANIFEST for Batch 2 has no genome version listed, we
# should only have data for samples in Batch 1. Perform all the checks
# against our trimmed list of expected counts.
self.assertEqual(Result.objects.count(), expected_counts['results'])
# Batches are now published..
self.assertEqual(Batch.objects.filter(published=True).count(),
expected_counts['batches'])
# Ensure the counts are accurate for each sample..
for ec in expected_counts['results_per_sample']:
sample = Sample.objects.get(name=ec['sample'],
batch__name=ec['batch'])
self.assertTrue(sample.published)
self.assertEqual(sample.count, ec['count'])
# Batches are created with the samples, but are unpublished
for pk, count in expected_counts['samples_per_batch']:
batch = Batch.objects.get(pk=pk)
self.assertTrue(batch.published)
self.assertEqual(batch.count, count)
def test_invalid_url(self):
"""
Tests creating jobs with URLs that are not
Publico's News
"""
response = self.api.post(
reverse("cm_url_search"),
{
"urls": [
"https://www.google.pt/",
# Notice that the following link is not a news
"https://www.cmjornal.pt/",
]
},
format="json",
)
# Assert that a `job_id` is returned
self.assertIn("job_id", response.data)
# Assert that a `results_url` is returned
self.assertIn("results_url", response.data)
# Make the worker dispatch all jobs in sync mode
get_worker().work(burst=True)
# Now make the request to get the results
response = self.api.get(response.data["results_url"])
# Assert that response is status code 200
self.assertEqual(
response.status_code,
status.HTTP_200_OK,
)
def setUp(self):
super(DeleteTestCase, self).setUp()
# Immediately validates and creates a sample
management.call_command('samples', 'queue')
# Synchronously work on queue
worker1 = get_worker('variants')
worker2 = get_worker('default')
# Work on variants...
worker1.work(burst=True)
# Work on effects...
worker2.work(burst=True)
# Create and record some data that will be used to create knowledge
# capture assessments later on.
self.pathogenicity = Pathogenicity(name='pathogenic')
self.pathogenicity.save()
self.parental_result = ParentalResult(name='heterozygous')
self.parental_result.save()
self.category = AssessmentCategory(name='other')
self.category.save()
self.user = User.objects.all()[0]
def test_anonymous_post(self):
url = reverse('workery_franchise_list_create_api_endpoint')
post_data = json.dumps({
"schema_name": "mikasoftware",
"postal_code": "n6j4x4",
"name": "Mika Software Corporation",
"alternate_name": "Mika Software",
"description": "An open source software company.",
"url": "https://mikasoftware.com",
"timezone_name": "America/Toronto",
"address_country": "Canada",
"address_locality": "London",
"address_region": "Ontario",
"postal_code": "N6J4X4",
"street_address": "120 Centre Street",
"street_address_extra": "Unit 102"
})
response = self.anon_client.post(url,
data=post_data,
content_type='application/json')
get_worker().work(
burst=True
) # Processes all BACKGROUND jobs in FOREGROUND then stop. (Note: https://stackoverflow.com/a/12273705)
self.assertIsNotNone(response)
self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
def test(self):
"Load a single VCF, reload the snpEff data using the same VCF."
management.call_command('samples', 'queue',
os.path.join(SAMPLE_DIRS[0], 'batch1/locus_1'))
# Synchronously work on queue
worker1 = get_worker('variants')
worker2 = get_worker('default')
worker1.work(burst=True)
worker2.work(burst=True)
expected_variant_effects_count = 5426
self.assertEqual(VariantEffect.objects.count(),
expected_variant_effects_count)
self.assertEqual(
VariantEffect.objects.aggregate(max_id=Max('id'))['max_id'],
expected_variant_effects_count)
management.call_command('variants', 'reload-snpeff',
os.path.join(SAMPLE_DIRS[0],
'batch1/locus_1/locus_1.vcf'))
# Ensure data was actually reloaded, check the auto-incremented key
self.assertEqual(VariantEffect.objects.count(),
expected_variant_effects_count)
# Since we reloaded, we should now have double the number of expected
# results, thus the 2 * operation in the assertion below.
self.assertEqual(
VariantEffect.objects.aggregate(max_id=Max('id'))['max_id'],
2 * expected_variant_effects_count)
def test_sending_mail_after_creating_team(self):
self.client.force_authenticate(user=self.user1)
data = create_valid_team_data()
response = self.client.post(self.url_team_list, data, format='json')
django_rq.get_worker().work(burst=True)
self.assertEqual(len(mail.outbox), len(data['members']) + 1)
def test_auto_albums(self):
'''make sure user can make auto albums, list and retrieve them'''
# make auto albums
auto_album_gen_res = self.client_users[0].get('/api/autoalbumgen/')
self.assertEqual(auto_album_gen_res.status_code, 200)
get_worker().work(burst=True)
# make sure auto albums are there
auto_album_list_res = self.client_users[0].get(
'/api/albums/auto/list/')
self.assertEqual(auto_album_list_res.status_code, 200)
# make sure user can retrieve each auto album
for album in auto_album_list_res.json()['results']:
auto_album_retrieve_res = self.client_users[0].get(
'/api/albums/auto/%d/' % album['id'])
self.assertEqual(auto_album_retrieve_res.status_code, 200)
self.assertTrue(len(auto_album_retrieve_res.json()['photos']) > 0)
# try making auto albums again and make sure there are no duplicates
num_auto_albums = len(auto_album_list_res.json()['results'])
auto_album_gen_res = self.client_users[0].get('/api/autoalbumgen/')
self.assertEqual(auto_album_gen_res.status_code, 200)
get_worker().work(burst=True)
auto_album_list_res = self.client_users[0].get(
'/api/albums/auto/list/')
self.assertEqual(
len(auto_album_list_res.json()['results']), num_auto_albums)
def test_run_once_job(self):
"""run an single run job with arguments and check if it gets deleted"""
test_job = Job.objects.get(
task='django_rq_jobs.tests.tasks.django_arg_check')
management.call_command('rqjobs')
get_worker('default').work(burst=True)
self.assertFalse(Job.objects.filter(pk=test_job.pk).exists())
def test_cant_retrieve_pending_connection_user_user_encircled_post_media_video(
self):
"""
should be able to retrieve an pending_connection_user_user encircled post media video
"""
user = make_user()
pending_connection_user_user = make_user()
pending_connection_user_user.connect_with_user_with_id(user_id=user.pk)
headers = make_authentication_headers_for_user(user=user)
test_video = get_test_video()
with open(test_video['path'], 'rb') as file:
file = File(file)
circle = make_circle(creator=pending_connection_user_user)
post = pending_connection_user_user.create_encircled_post(
video=file, circles_ids=[circle.pk])
get_worker('high', worker_class=SimpleWorker).work(burst=True)
url = self._get_url(post=post)
response = self.client.get(url, **headers, format='multipart')
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_can_retrieve_follower_user_post_media_video(self):
"""
should be able to retrieve an follower_user post media video
"""
user = make_user()
follower_user = make_user()
follower_user.follow_user(user=user)
headers = make_authentication_headers_for_user(user=user)
test_video = get_test_video()
with open(test_video['path'], 'rb') as file:
file = File(file)
post = follower_user.create_public_post(video=file)
get_worker('high', worker_class=SimpleWorker).work(burst=True)
url = self._get_url(post=post)
response = self.client.get(url, **headers, format='multipart')
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_media = json.loads(response.content)
post.refresh_from_db()
post_media = post.get_media().all()
self._compare_response_media_with_post_media(
post_media=post_media, response_media=response_media)
def test_cannot_retrieve_private_community_not_part_of_post_media_video(
self):
"""
should not be able to retrieve an private_community not part of post media video
"""
user = make_user()
community_creator = make_user()
private_community = make_community(
creator=community_creator, type=Community.COMMUNITY_TYPE_PRIVATE)
headers = make_authentication_headers_for_user(user=user)
test_video = get_test_video()
with open(test_video['path'], 'rb') as file:
file = File(file)
post = community_creator.create_community_post(
video=file, community_name=private_community.name)
get_worker('high', worker_class=SimpleWorker).work(burst=True)
url = self._get_url(post=post)
response = self.client.get(url, **headers, format='multipart')
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_sending_mail_after_deleting_team(self):
self.client.force_authenticate(user=self.user1)
first_number_of_mail = len(mail.outbox)
response = self.client.delete(self.url_team_detail)
django_rq.get_worker().work(burst=True)
self.assertEqual(len(mail.outbox), first_number_of_mail + len(self.data['members']) + 1)
def test_makemessages_django_tq_more_jobs(self):
queue = get_queue('default')
queue.enqueue(tasks.makemessages_task)
queue.enqueue(tasks.makemessages_task)
queue.enqueue(tasks.makemessages_task)
get_worker().work(burst=True)
def test_correct_job_response(self):
"""
Tests that the correct elements are returned when
fetching job results.
"""
response = self.api.post(
reverse("publico_url_search"),
{
"urls": [
"https://www.publico.pt/2021/01/31/economia/noticia/irs-contribuintes-podem-validar-agregado-familiar-ate-15-fevereiro-1948701"
],
},
)
# Assert that a `job_id` is returned
self.assertIn("job_id", response.data)
# Assert that a `results_url` is returned
self.assertIn("results_url", response.data)
# here we dispatch the worker so that job gets done in sync mode
get_worker().work(burst=True)
# Now make the request to get the results
response = self.api.get(response.data["results_url"])
self.assertIn("number_of_news", response.data)
self.assertIn("date", response.data)
# Check that date is (almost) equal to now
# maximum 1 sec diff
self.assertTrue(
abs(now() - datetime_from_string(
response.data["date"],
order="YMD",
)) < datetime.timedelta(seconds=1))
self.assertIn("news", response.data)
def rq_worker(request):
[queue.empty() for queue in django_rq.get_worker().queues]
worker = django_rq.get_worker()
yield worker
[queue.empty() for queue in django_rq.get_worker().queues]
def rq_worker(request):
[queue.empty() for queue in django_rq.get_worker().queues]
worker = django_rq.get_worker()
yield worker
[queue.empty() for queue in django_rq.get_worker().queues]
def test_sending_mail_after_updating_member_data(self):
url = reverse('api:member-detail', kwargs={'pk': Team.objects.get(name=self.data['name']).pk, 'member_pk': Member.objects.get(name=self.data['members'][0]['name']).pk})
self.client.force_authenticate(user=self.user1)
first_number_of_email = len(mail.outbox)
data = create_valid_member_data()
response = self.client.put(url, data, format='json')
django_rq.get_worker().work(burst=True)
self.assertEqual(len(mail.outbox), first_number_of_email + len(self.data['members']) + 1)
def test_sending_mail_after_updating_team_data(self):
self.client.force_authenticate(user=self.user1)
first_number_of_mail = len(mail.outbox)
self.assertEqual(first_number_of_mail, len(self.data['members']) + 1)
response = self.client.put(self.url_team_detail, self.data, format='json')
django_rq.get_worker().work(burst=True)
self.assertEqual(len(mail.outbox), first_number_of_mail + len(self.data['members']) + 1)
def test_rq(settings, image, instance_no_image):
settings.SIMPLEIMAGES_TRANSFORM_CALLER = 'django_rq.enqueue'
instance_no_image.image.save(image.name, image.django_file)
django_rq.get_worker().work(burst=True)
instance_no_image = instance_no_image.retrieve_from_database()
assert instance_no_image.thumbnail
def test_run_limited_job(self):
"""run a limited run job twice to see if it counts down and gets deleted"""
test_job = Job.objects.create(task='django_rq_jobs.tests.tasks.django_check', schedule_type=Job.HOURLY, repeats=2,
next_run=timezone.now() + timedelta(hours=-2))
management.call_command('rqjobs')
get_worker('default').work(burst=True)
self.assertEqual(Job.objects.get(pk=test_job.pk).repeats, 1)
management.call_command('rqjobs')
get_worker('default').work(burst=True)
self.assertFalse(Job.objects.filter(pk=test_job.pk).exists())
def test_sending_mail_after_deleting_member(self):
url = reverse('api:member-detail', kwargs={'pk': Team.objects.get(name=self.data['name']).pk, 'member_pk': Member.objects.get(name=self.data['members'][0]['name']).pk})
self.client.force_authenticate(user=self.user1)
first_number_of_mail = len(mail.outbox)
response = self.client.delete(url)
if len(self.data['members']) > settings.NUMBER_OF_MEMBERS[self.data['event']][0] - 1:
django_rq.get_worker().work(burst=True)
self.assertEqual(len(mail.outbox), first_number_of_mail + len(self.data['members']) + 1)
else:
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_rebuild_indices_using_subtasks(self):
result = rebuild_indices(self.SEARCH, subtask_indexing=True)
get_worker().work(burst=True)
self.assertIsNotNone(result)
self.assertIsInstance(result, dict)
self.index.refresh()
expected_index_count = self.published_entry_count + self.published_project_count
if not self.PUBLISH_FILTER_ENABLED:
expected_index_count += self.unpublished_entry_count + self.unpublished_project_count
self.assertEqual(expected_index_count, self.search.count())
def test_rebuild_indices_using_subtasks(self):
result = rebuild_indices(self.SEARCH, subtask_indexing=True)
get_worker().work(burst=True)
self.assertIsNotNone(result)
self.assertIsInstance(result, dict)
self.index.refresh()
expected_index_count = self.published_entry_count + self.published_project_count
if not self.PUBLISH_FILTER_ENABLED:
expected_index_count += self.unpublished_entry_count + self.unpublished_project_count
self.assertEqual(expected_index_count, self.search.count())
def test_run_limited_job(self):
"""run a limited run job twice to see if it counts down and gets deleted"""
test_job = Job.objects.create(
task='django_rq_jobs.tests.tasks.django_check',
schedule_type=Job.HOURLY,
repeats=2,
next_run=timezone.now() + timedelta(hours=-2))
management.call_command('rqjobs')
get_worker('default').work(burst=True)
self.assertEqual(Job.objects.get(pk=test_job.pk).repeats, 1)
management.call_command('rqjobs')
get_worker('default').work(burst=True)
self.assertFalse(Job.objects.filter(pk=test_job.pk).exists())
def begin_processing(self, schema_name, name, alternate_name, description,
country, city, province, street_number, street_name,
apartment_unit, street_type, street_type_other,
street_direction, postal_code, timezone_name,
police_report_url, default_position, default_zoom):
"""
Functin will create a new tenant based on the parameters.
"""
# Create your tenant
tenant = SharedOrganization(
schema_name=schema_name,
name=name,
alternate_name=alternate_name,
description=description,
country=country,
city=city,
province=province,
street_number=street_number,
street_name=street_name,
apartment_unit=apartment_unit,
street_type=street_type,
street_type_other=street_type_other,
street_direction=street_direction,
postal_code=postal_code,
timezone_name=timezone_name,
police_report_url=police_report_url,
default_position=default_position,
default_zoom=default_zoom,
)
tenant.save()
get_worker().work(
burst=True
) # Processes all BACKGROUND jobs in FOREGROUND then stop. (Note: https://stackoverflow.com/a/12273705
# Add one or more domains for the tenant
domain = SharedOrganizationDomain()
domain.domain = settings.NWAPP_BACKEND_HTTP_DOMAIN
domain.domain = tenant.schema_name + '.' + settings.NWAPP_BACKEND_HTTP_DOMAIN
domain.tenant = tenant
domain.is_primary = False
domain.save()
get_worker().work(
burst=True
) # Processes all BACKGROUND jobs in FOREGROUND then stop. (Note: https://stackoverflow.com/a/12273705
# Populate our new organization tenant with post-creation data.
call_command('populate_tenant_content', schema_name, verbosity=0)
def test_sending_mail_after_post(self):
self.client.force_authenticate(user=self.user1)
data = create_valid_member_data()
first_number_of_mail = len(mail.outbox)
response = self.client.post(self.url_member_list, data, format='json')
if len(self.data['members']) < settings.NUMBER_OF_MEMBERS[self.data['event']][1] - 1:
django_rq.get_worker().work(burst=True)
self.assertEqual(len(mail.outbox), first_number_of_mail + len(self.data['members']) + 2)
else:
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_create_media(self):
# We must import MediaAdmin here and not in the header
# otherwise it's loaded before settings and thus, fails.
from .admin import MediaAdmin
ma = MediaAdmin(Media, self.site)
request = self.factory.get('/admin/archives/media/add')
request.user = self.user
obj = MediaFactory.build()
# As obj is not saved, it hasn't got an id
self.assertEqual(obj.id, None)
ma.save_model(request, obj, None, False)
# Below, we ensure that object has been created as it has an id.
self.assertTrue(isinstance(obj.id, int))
get_worker().work(burst=True)
def test_run_job(self):
"""run a job and check if it's rescheduled properly"""
management.call_command('rqjobs')
get_worker('default').work(burst=True)
test_job = Job.objects.get(task='django_rq_jobs.tests.tasks.django_check', schedule_type=Job.HOURLY)
self.assertNotEqual(test_job, None)
self.assertNotEqual(test_job.rq_id, None)
self.assertNotEqual(test_job.rq_origin, None)
self.assertIsNot(test_job.rq_job, None)
self.assertNotEqual(test_job.rq_status(), None)
self.assertNotEqual(test_job.rq_origin, None)
self.assertNotEqual(test_job.rq_link(), None)
self.assertNotEqual(test_job.last_run, None)
self.assertTrue(test_job.next_run > timezone.now())
def setUp(self):
super(AlleleTestCase, self).setUp()
# Immediately validates and creates a sample
management.call_command("samples", "queue")
# Synchronously work on queue
worker1 = get_worker("variants")
worker2 = get_worker("default")
# Work on variants...
worker1.work(burst=True)
# Work on effects...
worker2.work(burst=True)
def test_get_worker_default(self):
"""
By default, ``get_worker`` should return worker for ``default`` queue.
"""
worker = get_worker()
queue = worker.queues[0]
self.assertEqual(queue.name, 'default')
def test_work(self, mocker):
close_database = mocker.patch(
"metadeploy.rq_worker.ConnectionClosingWorker.close_database")
worker = get_worker()
worker.work(burst=True)
assert close_database.called
def worker():
# Clear queue
q = django_rq.get_queue()
q.empty()
worker = django_rq.get_worker()
return worker
def test_request_force_stop__dead_horse(self, mocker):
kill = mocker.patch("os.kill", side_effect=OSError)
worker = get_worker()
job = mocker.MagicMock()
worker._horse_pid = 1
with pytest.raises(OSError):
worker.request_force_stop(signal.SIGTERM, None)
def test_media_encoded_state(self):
#we create media without encoding
self.media = MediaFactory.build(title='test_media_encoded_state', file__from_path=os.path.join(os.path.dirname(__file__), 'tests/data/audio-mini.mp3'))
self.media.save(encode=False)
#check it has been well created and encoding state is not encoded
self.assertEqual(self.media.id, 1)
self.assertEqual(self.media.encoding_state, ENCODING_NOT_ENCODED)
#put it in encoding queue
queue = django_rq.get_queue('default')
job = queue.enqueue(call_command, args=('encode', self.media.id))
#check job is in encoding queue
self.assertTrue(job.is_queued)
#start encoding
worker = get_worker('default')
worker.work(burst=True)
#check encoding state is encoded, job is now in finished status
self.assertEqual(self.media.encoding_state, ENCODING_ENCODED)
self.assertFalse(job.is_queued)
self.assertTrue(job.is_finished)
failed_queue = Queue(name='failed', connection=queue.connection)
self.assertFalse(job.id in failed_queue.job_ids)
def test_run_job(self):
"""run a job and check if it's rescheduled properly"""
management.call_command('rqjobs')
get_worker('default').work(burst=True)
test_job = Job.objects.get(
task='django_rq_jobs.tests.tasks.django_check',
schedule_type=Job.HOURLY)
self.assertNotEqual(test_job, None)
self.assertNotEqual(test_job.rq_id, None)
self.assertNotEqual(test_job.rq_origin, None)
self.assertIsNot(test_job.rq_job, None)
self.assertNotEqual(test_job.rq_status(), None)
self.assertNotEqual(test_job.rq_origin, None)
self.assertNotEqual(test_job.rq_link(), None)
self.assertNotEqual(test_job.last_run, None)
self.assertTrue(test_job.next_run > timezone.now())
def test_request_stop__current_job(self, mocker):
signal = mocker.patch("signal.signal")
worker = get_worker()
worker.get_current_job = mocker.MagicMock()
worker.request_stop(signal.SIGINT, None)
# worker flagged to stop
assert worker._stopped
def test_work(self, mocker):
close_database = mocker.patch(
"{{cookiecutter.project_slug}}.rq_worker.ConnectionClosingWorker.close_database"
)
worker = get_worker()
worker.work(burst=True)
assert close_database.called
def test_request_stop(self, mocker):
signal = mocker.patch("signal.signal")
worker = get_worker()
with pytest.raises(StopRequested):
worker.request_stop(signal.SIGINT, None)
# new signal handlers installed
assert signal.call_count == 2
def test_create_new_app_user(self):
data = {
'app_facebook_id': self.mobile_app.facebook_id,
'oauth_token': user_oauth_token,
'facebook_id': user_facebook_id
}
response = self.client.post('/appUserLogin/', data, format='json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
get_worker().work(burst=True) # Process all jobs, then stop
# Test that an AppUser was actually created
user = AppUser.objects.get(facebook_id=user_facebook_id)
self.assertEqual(isinstance(user, AppUser), True)
# Test that his friends were created
user_friends_count = user.friends.all().count()
self.assertEqual(user_friends_count, 4)
def test_close_database__good(self, mocker):
conn = MagicMock()
all_ = mocker.patch("django.db.connections.all")
all_.return_value = [conn]
worker = get_worker()
worker.close_database()
assert conn.close.called
def test_close_database__database_error__reraise(self, mocker):
conn = MagicMock()
conn.close.side_effect = DatabaseError("reraise me")
all_ = mocker.patch("django.db.connections.all")
all_.return_value = [conn]
worker = get_worker()
with pytest.raises(DatabaseError):
worker.close_database()
def test_create_new_app_user(self):
data = {
'app_facebook_id': self.mobile_app.facebook_id,
'oauth_token': user_oauth_token,
'facebook_id': user_facebook_id
}
response = self.client.post('/appUserLogin/', data, format='json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
get_worker().work(burst=True) # Process all jobs, then stop
# Test that an AppUser was actually created
user = AppUser.objects.get(facebook_id=user_facebook_id)
self.assertEqual(isinstance(user, AppUser), True)
# Test that his friends were created
user_friends_count = user.friends.all().count()
self.assertEqual(user_friends_count, 4)
def test_perform_job(self, mocker):
close_database = mocker.patch(
"statusite.worker.RequeueingWorker.close_database")
mocker.patch("rq.worker.Worker.perform_job")
worker = get_worker()
# Symbolic call only, since we've mocked out the super:
worker.perform_job(None, None)
assert close_database.called
def test_perform_job(self, mocker):
close_database = mocker.patch(
"metadeploy.rq_worker.ConnectionClosingWorker.close_database")
mocker.patch("rq.worker.Worker.perform_job")
worker = get_worker()
# Symbolic call only, since we've mocked out the super:
worker.perform_job(None, None)
assert close_database.called
def test_close_database__database_error__no_reraise(self, mocker):
conn = MagicMock()
conn.close.side_effect = DatabaseError("closed not connected don't reraise me")
all_ = mocker.patch("django.db.connections.all")
all_.return_value = [conn]
worker = get_worker()
worker.close_database()
assert conn.close.called
def test_close_database__interface_error(self, mocker):
conn = MagicMock()
conn.close.side_effect = InterfaceError()
all_ = mocker.patch("django.db.connections.all")
all_.return_value = [conn]
worker = get_worker()
worker.close_database()
assert conn.close.called
def startWorkers(self):
# Find the number of current workers
queues = getattr(settings, 'RQ_QUEUES', {})
default = queues['default'] if 'default' in queues else None
variants = queues['variants'] if 'variants' in queues else None
if not (queues and default and variants):
log.warning('RQ_QUEUES settings could not be found')
return
# Create connections to redis to identify the workers
def_connection = redis.Redis(host=default['HOST'],
port=default['PORT'],
db=default['DB'])
var_connection = redis.Redis(host=variants['HOST'],
port=variants['PORT'],
db=variants['DB'])
# Get all the workers connected with our redis server
try:
all_workers = Worker.all(def_connection) + \
Worker.all(var_connection)
except ConnectionError:
log.warning('Could not connect to redis server to create workers. '
'Please make sure Redis server is running')
return
found_default = False
found_variant = False
# Loop through all the workers (even duplicates)
for worker in all_workers:
found_default = found_default or 'default' in worker.queue_names()
found_variant = found_variant or 'variants' in worker.queue_names()
# Start the required worker
if not found_variant:
log.debug('Did not find variants worker. Starting ... ')
get_worker('variants').work(burst=True)
if not found_default:
log.debug('Did not find default worker. Starting ... ')
get_worker('default').work(burst=True)
def test_request_force_stop(self, mocker):
kill = mocker.patch("os.kill")
worker = get_worker()
job = mocker.MagicMock()
worker.get_current_job = mocker.MagicMock(return_value=job)
worker._horse_pid = 1
with pytest.raises(SystemExit):
worker.request_force_stop(signal.SIGTERM, None)
job.func.delay.assert_called_once()
kill.assert_called_once_with(1, signal.SIGKILL)
def test_wrong_genome_version(self):
# Immediately validates and creates a sample.
management.call_command('samples', 'queue')
# Synchronously work on queue.
worker1 = get_worker('variants')
worker2 = get_worker('default')
# Work on variants.
worker1.work(burst=True)
# Work on effects.
worker2.work(burst=True)
# Since the genome version was required but does not match any of the
# versions specified in the MANIFESTs, we should have no data.
self.assertEqual(Variant.objects.count(), 0)
self.assertEqual(Result.objects.count(), 0)
self.assertEqual(Sample.objects.count(), 0)
self.assertEqual(Cohort.objects.count(), 0)
self.assertEqual(Batch.objects.count(), 0)
self.assertEqual(Project.objects.count(), 0)
def test(self):
"Load a single VCF, reload the snpEff data using the same VCF."
management.call_command('samples', 'queue',
os.path.join(SAMPLE_DIRS[0], 'batch1/sample1'))
# Synchronously work on queue
worker1 = get_worker('variants')
worker2 = get_worker('default')
worker1.work(burst=True)
worker2.work(burst=True)
self.assertEqual(VariantEffect.objects.count(), 614)
self.assertEqual(
VariantEffect.objects.aggregate(max_id=Max('id'))['max_id'], 614)
management.call_command('variants', 'reload-snpeff',
os.path.join(SAMPLE_DIRS[0],
'batch1/sample1/results.vcf'))
# Ensure data was actually reloaded, check the auto-incremented key
self.assertEqual(VariantEffect.objects.count(), 614)
self.assertEqual(VariantEffect.objects.aggregate(
max_id=Max('id'))['max_id'], 614 * 2)
def test_run_once_job(self):
"""run an single run job with arguments and check if it gets deleted"""
test_job = Job.objects.get(task='django_rq_jobs.tests.tasks.django_arg_check')
management.call_command('rqjobs')
get_worker('default').work(burst=True)
self.assertFalse(Job.objects.filter(pk=test_job.pk).exists())
def run_jobs():
"""
Execute all the pending jobs.
"""
get_worker(settings.ASYNC_QUEUE).work(burst=True)
def createWorker():
worker = django_rq.get_worker()
worker.work()
def test_pipeline(self):
expected_counts = {
'batches': 2,
'cohorts': 2,
'genes': 65,
'projects': 1,
'results_per_sample': [
{
'batch': 'batch1',
'sample': 'NA12891',
'count': 1963,
},
{
'batch': 'batch1',
'sample': 'NA12892',
'count': 1963,
},
{
'batch': 'batch1',
'sample': 'NA12878',
'count': 1963,
},
{
'batch': 'batch2',
'sample': 'NA12891',
'count': 2094,
},
{
'batch': 'batch2',
'sample': 'NA12892',
'count': 2094,
},
{
'batch': 'batch2',
'sample': 'NA12878',
'count': 2094,
},
],
'samples': 6,
'transcripts': 108,
'variant_effects': 8788,
'variants': 4057,
'samples_per_batch': [(1, 3), (2, 3)],
}
expected_counts['results'] = \
sum([x['count'] for x in expected_counts['results_per_sample']])
# Immediately validates and creates a sample
management.call_command('samples', 'queue')
# Synchronously work on queue
worker1 = get_worker('variants')
worker2 = get_worker('default')
# Ensure sample-related entries are created..
self.assertEqual(Project.objects.count(), expected_counts['projects'])
self.assertEqual(Batch.objects.count(), expected_counts['batches'])
self.assertEqual(Sample.objects.count(), expected_counts['samples'])
# World and project cohort..
self.assertEqual(Cohort.objects.count(), expected_counts['cohorts'])
# Nothing published yet..
self.assertEqual(Sample.objects.filter(published=False).count(),
expected_counts['samples'])
self.assertEqual(
Cohort.objects.filter(count=0, published=False).count(),
expected_counts['cohorts'])
self.assertEqual(
Batch.objects.filter(count=0, published=False).count(),
expected_counts['batches'])
# Manifests are stored
self.assertEqual(SampleManifest.objects.count(),
expected_counts['samples'])
for manifest in SampleManifest.objects.all():
self.assertNotEqual(manifest.content, '')
self.assertFalse(manifest.content_has_changed())
# Work on variants...
worker1.work(burst=True)
self.assertEqual(Variant.objects.count(), expected_counts['variants'])
# Work on effects...
worker2.work(burst=True)
self.assertEqual(Gene.objects.count(), expected_counts['genes'])
self.assertEqual(Transcript.objects.count(),
expected_counts['transcripts'])
self.assertEqual(VariantEffect.objects.count(),
expected_counts['variant_effects'])
self.assertEqual(Sift.objects.count(), 0)
self.assertEqual(PolyPhen2.objects.count(), 0)
self.assertEqual(ThousandG.objects.count(), 0)
self.assertEqual(EVS.objects.count(), 0)
# Results loaded..
self.assertEqual(Result.objects.count(), expected_counts['results'])
# Batches are now published..
self.assertEqual(Batch.objects.filter(published=True).count(),
expected_counts['batches'])
# Ensure the counts are accurate for each sample..
for ec in expected_counts['results_per_sample']:
sample = Sample.objects.get(name=ec['sample'],
batch__name=ec['batch'])
self.assertTrue(sample.published)
self.assertEqual(sample.count, ec['count'])
# Batches are created with the samples, but are unpublished
for pk, count in expected_counts['samples_per_batch']:
batch = Batch.objects.get(pk=pk)
self.assertTrue(batch.published)
self.assertEqual(batch.count, count)
# Ensure the state changes were logged..
system = System.get(Sample.objects.all()[0])
self.assertEqual(len(system), 3)
def test_pipeline(self):
# Immediately validates and creates a sample
management.call_command('samples', 'queue')
# Synchronously work on queue
worker1 = get_worker('variants')
worker2 = get_worker('default')
# Ensure sample-related entries are created..
self.assertEqual(Project.objects.count(), 1)
self.assertEqual(Batch.objects.count(), 4)
self.assertEqual(Sample.objects.count(), 15)
# World and project cohort..
self.assertEqual(Cohort.objects.count(), 2)
# Nothing published yet..
self.assertEqual(Sample.objects.filter(published=False).count(), 15)
self.assertEqual(
Cohort.objects.filter(count=0, published=False).count(), 2)
self.assertEqual(
Batch.objects.filter(count=0, published=False).count(), 4)
# Manifests are stored
self.assertEqual(SampleManifest.objects.count(), 15)
for manifest in SampleManifest.objects.all():
self.assertNotEqual(manifest.content, '')
self.assertFalse(manifest.content_has_changed())
# Work on variants...
worker1.work(burst=True)
self.assertEqual(Variant.objects.count(), 674)
# Work on effects...
worker2.work(burst=True)
self.assertEqual(Gene.objects.count(), 104)
self.assertEqual(Transcript.objects.count(), 255)
self.assertEqual(VariantEffect.objects.count(), 1418)
self.assertEqual(Sift.objects.count(), 0)
self.assertEqual(PolyPhen2.objects.count(), 0)
self.assertEqual(ThousandG.objects.count(), 0)
self.assertEqual(EVS.objects.count(), 0)
# Results loaded..
self.assertEqual(Result.objects.count(), 3436)
# Batches are now published..
self.assertEqual(Batch.objects.filter(published=True).count(), 3)
# Ensure the counts are accurate for each sample..
for pk, count in [(1, 289), (2, 281), (3, 268), (4, 295), (5, 296),
(6, 293), (7, 264), (8, 289), (9, 264), (10, 293),
(11, 289), (12, 315)]:
sample = Sample.objects.get(pk=pk)
self.assertTrue(sample.published)
self.assertEqual(sample.count, count)
# Batches are created with the samples, but are unpublished
for pk, count in [(1, 5), (2, 3), (3, 4)]:
batch = Batch.objects.get(pk=pk)
self.assertTrue(batch.published)
self.assertEqual(batch.count, count)
# Ensure the state changes were logged..
system = System.get(Sample.objects.all()[0])
self.assertEqual(len(system), 3)
def process_jobs(self): from django_rq import get_worker get_worker().work(burst=True)
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "thresher_backend.settings")
# Do time consuming Django setup before forking individual worker processes.
import django
django.setup()
from django.conf import settings
from django_rq import get_worker
if __name__ == '__main__':
worker = get_worker('task_exporter', 'task_importer', 'file_importer',
'nlp_generator', 'nlp_importer')
worker.work()
