def setUpClass(cls):
cls.tempdir = tempfile.mkdtemp()
MPControl.shutdown()
MPControl.set_multiprocess_engine(cls.name)
# Create a wrapper for LocalCluster so that the HPC controller can be tested locally
# And then bind it so that it works in py27 right
def fake_cluster(*args, **kwargs):
replace_args = dict()
replace_args["n_workers"] = kwargs.pop("n_workers", 0)
replace_args["threads_per_worker"] = kwargs.pop(
"threads_per_worker", 1)
replace_args["processes"] = kwargs.pop("processes", True)
replace_args["local_dir"] = kwargs.pop("local_directory", None)
clust = distributed.LocalCluster(**replace_args)
clust._active_worker_n = 0
def _count_active_workers(self):
val = self._active_worker_n
self._active_worker_n += 1
return val
clust._count_active_workers = types.MethodType(
_count_active_workers, clust)
return clust
MPControl.client._cluster_controller_class = types.MethodType(
fake_cluster, MPControl.client)
MPControl.client.set_job_size_params(n_jobs=1,
n_cores_per_job=1,
mem_per_job="1gb")
MPControl.client._local_directory = cls.tempdir
MPControl.connect()
def test_multiprocessing_init(self):
MPControl.shutdown()
self.workflow.multiprocessing_controller = "local"
self.workflow.initialize_multiprocessing()
self.assertTrue(MPControl.is_initialized)
def tearDownClass(cls):
MPControl.shutdown()
MPControl.set_multiprocess_engine("local")
MPControl.connect()
if cls.tempdir is not None:
shutil.rmtree(cls.tempdir)
def setUpClass(cls):
cls.tempdir = tempfile.mkdtemp()
MPControl.shutdown()
MPControl.set_multiprocess_engine("dask-local")
MPControl.connect(local_dir=cls.tempdir, n_workers=1, processes=False)
def tearDownClass(cls):
MPControl.shutdown()
MPControl.set_multiprocess_engine("local")
MPControl.connect()
def setUpClass(cls):
MPControl.shutdown()
MPControl.client = None
def setUpClass(cls):
MPControl.shutdown()
MPControl.set_multiprocess_engine(cls.name)
MPControl.connect()
def setUpClass(cls):
cls.tempdir = tempfile.mkdtemp()
MPControl.shutdown()
MPControl.set_multiprocess_engine(cls.name)
MPControl.connect(local_dir=cls.tempdir, n_workers=1)
def setUpClass(cls):
cls.temp_dir = tempfile.mkdtemp()
cls.server = kvsstcp.KVSServer("", 0)
MPControl.shutdown()
MPControl.set_multiprocess_engine(cls.name)
MPControl.connect(host=cls.server.cinfo[0], port=cls.server.cinfo[1])
