def test_chunker_list(self): self.assertEqual( list(BulkImportParallelRunner.chunker_list([1, 2, 3], 3)), [[1], [2], [3]]) self.assertEqual( list(BulkImportParallelRunner.chunker_list([1, 2, 3], 2)), [[1, 3], [2]]) self.assertEqual( list(BulkImportParallelRunner.chunker_list([1, 2, 3], 1)), [[1, 2, 3]])
def test_update_elapsed_seconds(self, redis_service_mock): redis_service_mock.return_value = Mock() importer = BulkImportParallelRunner( open( os.path.join(os.path.dirname(__file__), '..', 'samples/sample_ocldev.json'), 'r').read(), 'ocladmin', True) self.assertIsNotNone(importer.start_time) self.assertEqual(importer.elapsed_seconds, 0) importer.update_elapsed_seconds() self.assertTrue(importer.elapsed_seconds > 0)
def test_get_overall_tasks_progress(self, redis_service_mock): redis_instance_mock = Mock() redis_instance_mock.get_int.side_effect = [100, 50] redis_service_mock.return_value = redis_instance_mock importer = BulkImportParallelRunner( open( os.path.join(os.path.dirname(__file__), '..', 'samples/sample_ocldev.json'), 'r').read(), 'ocladmin', True) self.assertEqual(importer.get_overall_tasks_progress(), 0) importer.tasks = [Mock(task_id='task1'), Mock(task_id='task2')] self.assertEqual(importer.get_overall_tasks_progress(), 150)
def test_make_parts(self, redis_service_mock): redis_service_mock.return_value = Mock() importer = BulkImportParallelRunner( open( os.path.join(os.path.dirname(__file__), '..', 'samples/sample_ocldev.json'), 'r').read(), 'ocladmin', True) self.assertEqual(len(importer.parts), 7) self.assertEqual(len(importer.parts[0]), 2) self.assertEqual(len(importer.parts[1]), 2) self.assertEqual(len(importer.parts[2]), 1) self.assertEqual(len(importer.parts[3]), 23) self.assertEqual(len(importer.parts[4]), 22) self.assertEqual(len(importer.parts[5]), 2) self.assertEqual(len(importer.parts[6]), 12) self.assertEqual([l['type'] for l in importer.parts[0]], ['Organization', 'Organization']) self.assertEqual([l['type'] for l in importer.parts[1]], ['Source', 'Source']) self.assertEqual([l['type'] for l in importer.parts[2]], ['Source Version']) self.assertEqual(list({l['type'] for l in importer.parts[3]}), ['Concept']) self.assertEqual(list({l['type'] for l in importer.parts[4]}), ['Mapping']) self.assertEqual([l['type'] for l in importer.parts[5]], ['Source Version', 'Source Version']) self.assertEqual(list({l['type'] for l in importer.parts[6]}), ['Concept'])
def bulk_import_parallel_inline(self, to_import, username, update_if_exists, threads=5): from core.importers.models import BulkImportParallelRunner return BulkImportParallelRunner(content=to_import, username=username, update_if_exists=update_if_exists, parallel=threads, self_task_id=self.request.id).run()
def test_notify_progress(self, redis_service_mock): # pylint: disable=no-self-use redis_instance_mock = Mock(set_json=Mock()) redis_service_mock.return_value = redis_instance_mock importer = BulkImportParallelRunner( open( os.path.join(os.path.dirname(__file__), '..', 'samples/sample_ocldev.json'), 'r').read(), 'ocladmin', True, None, 'task-id') importer.tasks = [Mock(task_id='task-1'), Mock(task_id='task-2')] now = 1607346541.793877 # datetime.datetime(2020, 12, 7, 13, 09, 1, 793877) UTC importer.start_time = now importer.elapsed_seconds = 10.45 importer.notify_progress() redis_instance_mock.set_json.assert_called_once_with( 'task-id', dict( summary= "Started: 2020-12-07 13:09:01.793877 | Processed: 0/64 | Time: 10.45secs", #sub_task_ids=['task-1', 'task-2'] ))
def test_is_any_process_alive(self, redis_service_mock): redis_service_mock.return_value = Mock() importer = BulkImportParallelRunner( open( os.path.join(os.path.dirname(__file__), '..', 'samples/sample_ocldev.json'), 'r').read(), 'ocladmin', True) self.assertFalse(importer.is_any_process_alive()) importer.groups = [ Mock(completed_count=Mock(return_value=5), __len__=Mock(return_value=5)), Mock(completed_count=Mock(return_value=5), __len__=Mock(return_value=5)), ] self.assertFalse(importer.is_any_process_alive()) importer.groups = [ Mock(completed_count=Mock(return_value=10), __len__=Mock(return_value=10)), Mock(completed_count=Mock(return_value=5), __len__=Mock(return_value=10)), ] self.assertTrue(importer.is_any_process_alive()) importer.groups = [ Mock(completed_count=Mock(return_value=5), __len__=Mock(return_value=10)), Mock(completed_count=Mock(return_value=5), __len__=Mock(return_value=10)), ] self.assertTrue(importer.is_any_process_alive()) importer.groups = [ Mock(completed_count=Mock(return_value=0), __len__=Mock(return_value=10)), ] self.assertTrue(importer.is_any_process_alive()) importer.groups = [ Mock(completed_count=Mock(return_value=9), __len__=Mock(return_value=10)), ] self.assertTrue(importer.is_any_process_alive())