コード例 #1
0
 def test_spawned_children_exception(self):
     vids = list(range(2))
     rt = Runtime(get_spawned_process_pool_factory())
     _, vids_with_err = rt.run(TestRuntime.query_that_throws_at_0,
                               vids,
                               print_error=False)
     self.assertEqual([0], vids_with_err)
コード例 #2
0
 def test_iterator_error(self):
     vids = list(range(2))
     rt = Runtime(get_spawned_process_pool_factory())
     gen = rt.get_result_iterator(TestRuntime.query_that_throws_at_0,
                                  vids,
                                  print_error=False)
     result = next(gen)
     self.assertIntervalSetEq(result,
                              TestRuntime.query_that_throws_at_0([1]))
     with self.assertRaises(RekallRuntimeException):
         next(gen)
コード例 #3
0
def get_runtime_for_script(num_workers=mp.cpu_count()):
    """Returns a parallel runtime that is safe for multithreaded programs.

    Notes:
        This uses SpawnedProcessPool and it does not work in a Jupyter
        Notebook. It will hang in Jupyter.
    """
    return Runtime(get_worker_pool_factory_for_script(num_workers))
コード例 #4
0
def get_runtime_for_jupyter(num_workers=mp.cpu_count()):
    """Returns a parallel runtime that works in Jupyter Notebook.
    
    Notes:
        This uses ForkedProcessPool so the usual caveats of forking a
        multithreaded program apply.
    """
    return Runtime(get_worker_pool_factory_for_jupyter(num_workers))
コード例 #5
0
    def test_exception_inline(self):
        vids = list(range(2))
        rt = Runtime.inline()

        _, vids_with_err = rt.run(TestRuntime.query_that_throws_at_0,
                                  vids,
                                  print_error=False)
        self.assertEqual([0], vids_with_err)
コード例 #6
0
def get_runtime_for_ipython_cluster(client):
    """Returns a runtime using an ipython cluster as worker processes. 
    
    This is the best option in a Jupyter Notebook.
    """
    return Runtime(get_worker_pool_factory_for_ipython_cluster(client))
コード例 #7
0
 def test_returning_intervalset(self):
     vids = list(range(1, 101))
     rt = Runtime(get_spawned_process_pool_factory())
     answer, _ = rt.run(TestRuntime.query_that_throws_at_0, vids)
     self.assertIntervalSetEq(answer,
                              TestRuntime.query_that_throws_at_0(vids))
コード例 #8
0
 def test_spawned_children(self):
     vids = list(range(10))
     rt = Runtime(get_spawned_process_pool_factory())
     self.assertCollectionEq(
         rt.run(TestRuntime.query, vids, chunksize=3)[0],
         TestRuntime.query(vids))
コード例 #9
0
    def test_single_process_runtime(self):
        vids = list(range(1000))
        rt = Runtime.inline()

        self.assertCollectionEq(
            rt.run(TestRuntime.query, vids)[0], TestRuntime.query(vids))
コード例 #10
0
 def test_all_tasks_fail(self):
     vids = list(range(1))
     rt = Runtime.inline()
     with self.assertRaises(RekallRuntimeException):
         rt.run(TestRuntime.query_that_throws_at_0, vids, print_error=False)
コード例 #11
0
 def test_inline_iterator(self):
     vids = list(range(1000))
     rt = Runtime.inline()
     gen = rt.get_result_iterator(TestRuntime.query, vids, randomize=True)
     for result in gen:
         self.assertCollectionEq(result, TestRuntime.query(result.keys()))
コード例 #12
0
 def test_iterator(self):
     vids = list(range(1000))
     rt = Runtime(get_forked_process_pool_factory(5))
     gen = rt.get_result_iterator(TestRuntime.query, vids, randomize=False)
     for vid, result in zip(vids, gen):
         self.assertCollectionEq(result, TestRuntime.query([vid]))