def set_up_dask(n_jobs=2): MPControl.set_multiprocess_engine("dask-cluster") MPControl.client.use_default_configuration("rusty_ccb", n_jobs=n_jobs) MPControl.client.add_worker_conda( "source ~/.local/anaconda3/bin/activate inferelator") MPControl.client.add_slurm_command_line("--constraint=broadwell") MPControl.connect()
def setUpClass(cls): if not MPControl.is_initialized: MPControl.set_multiprocess_engine("local") MPControl.connect() DotProduct.set_mkl(True)
def initialize_multiprocessing(self): """ Register the multiprocessing controller if set and run .connect() """ if self.multiprocessing_controller is not None: MPControl.set_multiprocess_engine(self.multiprocessing_controller) MPControl.connect()
def setUpClass(cls): cls.tempdir = tempfile.mkdtemp() MPControl.shutdown() MPControl.set_multiprocess_engine(cls.name) # Create a wrapper for LocalCluster so that the HPC controller can be tested locally # And then bind it so that it works in py27 right def fake_cluster(*args, **kwargs): replace_args = dict() replace_args["n_workers"] = kwargs.pop("n_workers", 0) replace_args["threads_per_worker"] = kwargs.pop( "threads_per_worker", 1) replace_args["processes"] = kwargs.pop("processes", True) replace_args["local_dir"] = kwargs.pop("local_directory", None) clust = distributed.LocalCluster(**replace_args) clust._active_worker_n = 0 def _count_active_workers(self): val = self._active_worker_n self._active_worker_n += 1 return val clust._count_active_workers = types.MethodType( _count_active_workers, clust) return clust MPControl.client._cluster_controller_class = types.MethodType( fake_cluster, MPControl.client) MPControl.client.set_job_size_params(n_jobs=1, n_cores_per_job=1, mem_per_job="1gb") MPControl.client._local_directory = cls.tempdir MPControl.connect()
def start_mpcontrol_dask(n_cores=N_CORES): utils.Debug.set_verbose_level(1) MPControl.set_multiprocess_engine("dask-cluster") MPControl.client.minimum_cores = n_cores MPControl.client.maximum_cores = n_cores MPControl.client.walltime = '48:00:00' MPControl.client.add_worker_env_line('module load slurm') MPControl.client.add_worker_env_line('module load gcc/8.3.0') MPControl.client.add_worker_env_line('source ' + CONDA_ACTIVATE_PATH) MPControl.client.cluster_controller_options.append("-p ccb") MPControl.connect()
def test_cant_change(self): with self.assertRaises(RuntimeError): MPControl.set_multiprocess_engine("local")
DATA_FILE = "YPD_LOCAL_OPT_PV_DEXFORM.h5ad" CONDA_ACTIVATE_PATH = '~/.local/anaconda3/bin/activate' EXPRESSION_MATRIX_METADATA = ['Genotype', 'Genotype_Group', 'Replicate', 'Condition', 'tenXBarcode'] YEASTRACT_PRIOR = "YEASTRACT_20190713_BOTH.tsv" TF_NAMES = "tf_names_gold_standard.txt" YEASTRACT_TF_NAMES = "tf_names_yeastract.txt" INPUT_DIR = '/mnt/ceph/users/cjackson/inferelator/data/yeast' OUTPUT_PATH = '/mnt/ceph/users/cjackson/jackson_2019_inferelator_v050/' if __name__ == '__main__': MPControl.set_multiprocess_engine("dask-cluster") MPControl.client.use_default_configuration("rusty_preempt") MPControl.client.set_job_size_params(n_jobs=1) MPControl.client.add_worker_conda("source ~/.local/anaconda3/bin/activate inferelator") MPControl.connect() wkf = inferelator_workflow("stars", VelocityWorkflow) wkf.set_file_paths(input_dir=INPUT_DIR, output_dir=os.path.join(OUTPUT_PATH, "tau43"), gold_standard_file='gold_standard.tsv', priors_file=YEASTRACT_PRIOR, tf_names_file=YEASTRACT_TF_NAMES) wkf.set_expression_file(h5ad=DATA_FILE, h5_layer="smooth_count") wkf.set_velocity_parameters(velocity_file_name=DATA_FILE, velocity_file_type="h5ad", velocity_file_layer="pv") wkf.set_crossvalidation_parameters(split_gold_standard_for_crossvalidation=True, cv_split_ratio=0.5)
def tearDownClass(cls): MPControl.shutdown() MPControl.set_multiprocess_engine("local") MPControl.connect() if cls.tempdir is not None: shutil.rmtree(cls.tempdir)
def setUpClass(cls): cls.tempdir = tempfile.mkdtemp() MPControl.shutdown() MPControl.set_multiprocess_engine("dask-local") MPControl.connect(local_dir=cls.tempdir, n_workers=1, processes=False)
def set_up_dask(n_jobs=1): MPControl.set_multiprocess_engine("dask-cluster") MPControl.client.use_default_configuration("rusty_rome", n_jobs=n_jobs) MPControl.client.add_worker_conda("source ~/.local/anaconda3/bin/activate inferelator") MPControl.client.set_job_size_params(walltime="168:00:00") MPControl.connect()
def setUpClass(cls): cls.temp_dir = tempfile.mkdtemp() cls.server = kvsstcp.KVSServer("", 0) MPControl.shutdown() MPControl.set_multiprocess_engine(cls.name) MPControl.connect(host=cls.server.cinfo[0], port=cls.server.cinfo[1])
PRIORS_FILE_NAME = 'gold_standard.tsv.gz' GOLD_STANDARD_FILE_NAME = 'gold_standard.tsv.gz' META_DATA_FILE_NAME = 'meta_data.tsv' TF_LIST_FILE_NAME = 'tf_names.tsv' CV_SEEDS = list(range(42, 52)) # Multiprocessing uses the pathos implementation of multiprocessing (with dill instead of cPickle) # This is suited for a single computer but will not work on a distributed cluster n_cores_local = 10 local_engine = True # Multiprocessing needs to be protected with the if __name__ == 'main' pragma if __name__ == '__main__' and local_engine: MPControl.set_multiprocess_engine("multiprocessing") MPControl.client.set_processes(n_cores_local) MPControl.connect() # Define the general run parameters def set_up_workflow(wkf): wkf.set_file_paths(input_dir=DATA_DIR, output_dir=OUTPUT_DIR, tf_names_file=TF_LIST_FILE_NAME, meta_data_file=META_DATA_FILE_NAME, priors_file=PRIORS_FILE_NAME, gold_standard_file=GOLD_STANDARD_FILE_NAME) wkf.set_expression_file(tsv=EXPRESSION_FILE_NAME) wkf.set_file_properties(expression_matrix_columns_are_genes=False) wkf.set_run_parameters(num_bootstraps=5)
def test_bad_engine_II(self): with self.assertRaises(ValueError): MPControl.set_multiprocess_engine(unittest.TestCase)
def test_bad_engine(self): with self.assertRaises(ValueError): MPControl.set_multiprocess_engine("V8")
def tearDownClass(cls): MPControl.shutdown() MPControl.set_multiprocess_engine("local") MPControl.connect()
def setUpClass(cls): MPControl.shutdown() MPControl.set_multiprocess_engine(cls.name) MPControl.connect()
def setUpClass(cls): cls.tempdir = tempfile.mkdtemp() MPControl.shutdown() MPControl.set_multiprocess_engine(cls.name) MPControl.connect(local_dir=cls.tempdir, n_workers=1)
from inferelator.distributed.inferelator_mp import MPControl MPControl.set_multiprocess_engine("local") MPControl.connect()
def start_mpcontrol_dask(n_cores=N_CORES): MPControl.set_multiprocess_engine("dask-cluster") MPControl.client.use_default_configuration("rusty_ccb", n_jobs=2) MPControl.client.add_worker_conda( "source /mnt/ceph/users/sysbio/.anaconda3/bin/activate inferelator") MPControl.connect()