def test_chunker_list(self):
self.assertEqual(
list(BulkImportParallelRunner.chunker_list([1, 2, 3], 3)),
[[1], [2], [3]])
self.assertEqual(
list(BulkImportParallelRunner.chunker_list([1, 2, 3], 2)),
[[1, 3], [2]])
self.assertEqual(
list(BulkImportParallelRunner.chunker_list([1, 2, 3], 1)),
[[1, 2, 3]])
def test_update_elapsed_seconds(self, redis_service_mock):
redis_service_mock.return_value = Mock()
importer = BulkImportParallelRunner(
open(
os.path.join(os.path.dirname(__file__), '..',
'samples/sample_ocldev.json'), 'r').read(),
'ocladmin', True)
self.assertIsNotNone(importer.start_time)
self.assertEqual(importer.elapsed_seconds, 0)
importer.update_elapsed_seconds()
self.assertTrue(importer.elapsed_seconds > 0)
def test_get_overall_tasks_progress(self, redis_service_mock):
redis_instance_mock = Mock()
redis_instance_mock.get_int.side_effect = [100, 50]
redis_service_mock.return_value = redis_instance_mock
importer = BulkImportParallelRunner(
open(
os.path.join(os.path.dirname(__file__), '..',
'samples/sample_ocldev.json'), 'r').read(),
'ocladmin', True)
self.assertEqual(importer.get_overall_tasks_progress(), 0)
importer.tasks = [Mock(task_id='task1'), Mock(task_id='task2')]
self.assertEqual(importer.get_overall_tasks_progress(), 150)
def test_make_parts(self, redis_service_mock):
redis_service_mock.return_value = Mock()
importer = BulkImportParallelRunner(
open(
os.path.join(os.path.dirname(__file__), '..',
'samples/sample_ocldev.json'), 'r').read(),
'ocladmin', True)
self.assertEqual(len(importer.parts), 7)
self.assertEqual(len(importer.parts[0]), 2)
self.assertEqual(len(importer.parts[1]), 2)
self.assertEqual(len(importer.parts[2]), 1)
self.assertEqual(len(importer.parts[3]), 23)
self.assertEqual(len(importer.parts[4]), 22)
self.assertEqual(len(importer.parts[5]), 2)
self.assertEqual(len(importer.parts[6]), 12)
self.assertEqual([l['type'] for l in importer.parts[0]],
['Organization', 'Organization'])
self.assertEqual([l['type'] for l in importer.parts[1]],
['Source', 'Source'])
self.assertEqual([l['type'] for l in importer.parts[2]],
['Source Version'])
self.assertEqual(list({l['type']
for l in importer.parts[3]}), ['Concept'])
self.assertEqual(list({l['type']
for l in importer.parts[4]}), ['Mapping'])
self.assertEqual([l['type'] for l in importer.parts[5]],
['Source Version', 'Source Version'])
self.assertEqual(list({l['type']
for l in importer.parts[6]}), ['Concept'])
def bulk_import_parallel_inline(self,
to_import,
username,
update_if_exists,
threads=5):
from core.importers.models import BulkImportParallelRunner
return BulkImportParallelRunner(content=to_import,
username=username,
update_if_exists=update_if_exists,
parallel=threads,
self_task_id=self.request.id).run()
def test_notify_progress(self, redis_service_mock): # pylint: disable=no-self-use
redis_instance_mock = Mock(set_json=Mock())
redis_service_mock.return_value = redis_instance_mock
importer = BulkImportParallelRunner(
open(
os.path.join(os.path.dirname(__file__), '..',
'samples/sample_ocldev.json'), 'r').read(),
'ocladmin', True, None, 'task-id')
importer.tasks = [Mock(task_id='task-1'), Mock(task_id='task-2')]
now = 1607346541.793877 # datetime.datetime(2020, 12, 7, 13, 09, 1, 793877) UTC
importer.start_time = now
importer.elapsed_seconds = 10.45
importer.notify_progress()
redis_instance_mock.set_json.assert_called_once_with(
'task-id',
dict(
summary=
"Started: 2020-12-07 13:09:01.793877 | Processed: 0/64 | Time: 10.45secs",
#sub_task_ids=['task-1', 'task-2']
))
def test_is_any_process_alive(self, redis_service_mock):
redis_service_mock.return_value = Mock()
importer = BulkImportParallelRunner(
open(
os.path.join(os.path.dirname(__file__), '..',
'samples/sample_ocldev.json'), 'r').read(),
'ocladmin', True)
self.assertFalse(importer.is_any_process_alive())
importer.groups = [
Mock(completed_count=Mock(return_value=5),
__len__=Mock(return_value=5)),
Mock(completed_count=Mock(return_value=5),
__len__=Mock(return_value=5)),
]
self.assertFalse(importer.is_any_process_alive())
importer.groups = [
Mock(completed_count=Mock(return_value=10),
__len__=Mock(return_value=10)),
Mock(completed_count=Mock(return_value=5),
__len__=Mock(return_value=10)),
]
self.assertTrue(importer.is_any_process_alive())
importer.groups = [
Mock(completed_count=Mock(return_value=5),
__len__=Mock(return_value=10)),
Mock(completed_count=Mock(return_value=5),
__len__=Mock(return_value=10)),
]
self.assertTrue(importer.is_any_process_alive())
importer.groups = [
Mock(completed_count=Mock(return_value=0),
__len__=Mock(return_value=10)),
]
self.assertTrue(importer.is_any_process_alive())
importer.groups = [
Mock(completed_count=Mock(return_value=9),
__len__=Mock(return_value=10)),
]
self.assertTrue(importer.is_any_process_alive())