Ejemplo n.º 1
0
 def test_chunker_list(self):
     self.assertEqual(
         list(BulkImportParallelRunner.chunker_list([1, 2, 3], 3)),
         [[1], [2], [3]])
     self.assertEqual(
         list(BulkImportParallelRunner.chunker_list([1, 2, 3], 2)),
         [[1, 3], [2]])
     self.assertEqual(
         list(BulkImportParallelRunner.chunker_list([1, 2, 3], 1)),
         [[1, 2, 3]])
Ejemplo n.º 2
0
    def test_update_elapsed_seconds(self, redis_service_mock):
        redis_service_mock.return_value = Mock()

        importer = BulkImportParallelRunner(
            open(
                os.path.join(os.path.dirname(__file__), '..',
                             'samples/sample_ocldev.json'), 'r').read(),
            'ocladmin', True)
        self.assertIsNotNone(importer.start_time)
        self.assertEqual(importer.elapsed_seconds, 0)
        importer.update_elapsed_seconds()
        self.assertTrue(importer.elapsed_seconds > 0)
Ejemplo n.º 3
0
 def test_get_overall_tasks_progress(self, redis_service_mock):
     redis_instance_mock = Mock()
     redis_instance_mock.get_int.side_effect = [100, 50]
     redis_service_mock.return_value = redis_instance_mock
     importer = BulkImportParallelRunner(
         open(
             os.path.join(os.path.dirname(__file__), '..',
                          'samples/sample_ocldev.json'), 'r').read(),
         'ocladmin', True)
     self.assertEqual(importer.get_overall_tasks_progress(), 0)
     importer.tasks = [Mock(task_id='task1'), Mock(task_id='task2')]
     self.assertEqual(importer.get_overall_tasks_progress(), 150)
Ejemplo n.º 4
0
    def test_make_parts(self, redis_service_mock):
        redis_service_mock.return_value = Mock()

        importer = BulkImportParallelRunner(
            open(
                os.path.join(os.path.dirname(__file__), '..',
                             'samples/sample_ocldev.json'), 'r').read(),
            'ocladmin', True)

        self.assertEqual(len(importer.parts), 7)
        self.assertEqual(len(importer.parts[0]), 2)
        self.assertEqual(len(importer.parts[1]), 2)
        self.assertEqual(len(importer.parts[2]), 1)
        self.assertEqual(len(importer.parts[3]), 23)
        self.assertEqual(len(importer.parts[4]), 22)
        self.assertEqual(len(importer.parts[5]), 2)
        self.assertEqual(len(importer.parts[6]), 12)
        self.assertEqual([l['type'] for l in importer.parts[0]],
                         ['Organization', 'Organization'])
        self.assertEqual([l['type'] for l in importer.parts[1]],
                         ['Source', 'Source'])
        self.assertEqual([l['type'] for l in importer.parts[2]],
                         ['Source Version'])
        self.assertEqual(list({l['type']
                               for l in importer.parts[3]}), ['Concept'])
        self.assertEqual(list({l['type']
                               for l in importer.parts[4]}), ['Mapping'])
        self.assertEqual([l['type'] for l in importer.parts[5]],
                         ['Source Version', 'Source Version'])
        self.assertEqual(list({l['type']
                               for l in importer.parts[6]}), ['Concept'])
Ejemplo n.º 5
0
def bulk_import_parallel_inline(self,
                                to_import,
                                username,
                                update_if_exists,
                                threads=5):
    from core.importers.models import BulkImportParallelRunner
    return BulkImportParallelRunner(content=to_import,
                                    username=username,
                                    update_if_exists=update_if_exists,
                                    parallel=threads,
                                    self_task_id=self.request.id).run()
Ejemplo n.º 6
0
    def test_notify_progress(self, redis_service_mock):  # pylint: disable=no-self-use
        redis_instance_mock = Mock(set_json=Mock())
        redis_service_mock.return_value = redis_instance_mock
        importer = BulkImportParallelRunner(
            open(
                os.path.join(os.path.dirname(__file__), '..',
                             'samples/sample_ocldev.json'), 'r').read(),
            'ocladmin', True, None, 'task-id')
        importer.tasks = [Mock(task_id='task-1'), Mock(task_id='task-2')]
        now = 1607346541.793877  # datetime.datetime(2020, 12, 7, 13, 09, 1, 793877) UTC
        importer.start_time = now
        importer.elapsed_seconds = 10.45
        importer.notify_progress()

        redis_instance_mock.set_json.assert_called_once_with(
            'task-id',
            dict(
                summary=
                "Started: 2020-12-07 13:09:01.793877 | Processed: 0/64 | Time: 10.45secs",
                #sub_task_ids=['task-1', 'task-2']
            ))
Ejemplo n.º 7
0
    def test_is_any_process_alive(self, redis_service_mock):
        redis_service_mock.return_value = Mock()
        importer = BulkImportParallelRunner(
            open(
                os.path.join(os.path.dirname(__file__), '..',
                             'samples/sample_ocldev.json'), 'r').read(),
            'ocladmin', True)
        self.assertFalse(importer.is_any_process_alive())

        importer.groups = [
            Mock(completed_count=Mock(return_value=5),
                 __len__=Mock(return_value=5)),
            Mock(completed_count=Mock(return_value=5),
                 __len__=Mock(return_value=5)),
        ]
        self.assertFalse(importer.is_any_process_alive())

        importer.groups = [
            Mock(completed_count=Mock(return_value=10),
                 __len__=Mock(return_value=10)),
            Mock(completed_count=Mock(return_value=5),
                 __len__=Mock(return_value=10)),
        ]
        self.assertTrue(importer.is_any_process_alive())

        importer.groups = [
            Mock(completed_count=Mock(return_value=5),
                 __len__=Mock(return_value=10)),
            Mock(completed_count=Mock(return_value=5),
                 __len__=Mock(return_value=10)),
        ]
        self.assertTrue(importer.is_any_process_alive())

        importer.groups = [
            Mock(completed_count=Mock(return_value=0),
                 __len__=Mock(return_value=10)),
        ]
        self.assertTrue(importer.is_any_process_alive())

        importer.groups = [
            Mock(completed_count=Mock(return_value=9),
                 __len__=Mock(return_value=10)),
        ]
        self.assertTrue(importer.is_any_process_alive())