def generate_trees(smcogs_dir: str, genes_within_clusters: List[CDSFeature], nrpspks_genes: List[CDSFeature]) -> Dict[str, str]: """ smCOG phylogenetic tree construction """ pks_nrps_cds_names = set(feature.get_name() for feature in nrpspks_genes) logging.info("Calculating and drawing phylogenetic trees of cluster genes " "with smCOG members") cds_features = [] for cds in genes_within_clusters: cds_name = cds.get_name() if cds_name in pks_nrps_cds_names: continue if not cds.gene_functions.get_by_tool("smcogs"): continue cds_features.append(cds) with TemporaryDirectory(change=True): args = [] for index, cds in enumerate(cds_features): smcog = cds.gene_functions.get_by_tool("smcogs")[0].description.split(":")[0] args.append([cds, index, smcog, smcogs_dir]) subprocessing.parallel_function(smcog_tree_analysis, args) files = glob.glob("*.png") tree_filenames = {} for filename in files: tag = filename.rsplit(".png", 1)[0] tree_filenames[tag] = filename return tree_filenames
def test_timeout(self): start = time.time() timeout = 1 with self.assertRaisesRegex(RuntimeError, "Timeout in parallel .*"): subprocessing.parallel_function(time.sleep, [[5]]*2, timeout=timeout) elapsed = time.time() - start assert elapsed < 1.5
def generate_trees(smcogs_dir: str, hmm_results: Dict[str, List[HSP]], genes_within_clusters: List[CDSFeature], nrpspks_genes: List[CDSFeature]) -> Dict[str, str]: """ smCOG phylogenetic tree construction """ pks_nrps_gene_names = set( [feature.get_name() for feature in nrpspks_genes]) logging.info("Calculating and drawing phylogenetic trees of cluster genes " "with smCOG members") with TemporaryDirectory(change=True): cds_features = [] for cds in genes_within_clusters: gene_id = cds.get_name() if gene_id not in pks_nrps_gene_names and hmm_results.get(gene_id): cds_features.append(cds) args = [] for index, cds in enumerate(cds_features): smcog = hmm_results[cds.get_name()][0].hit_id.split(":")[0] args.append([cds, index, smcog, smcogs_dir]) subprocessing.parallel_function(smcog_tree_analysis, args) files = glob.glob("*.png") tree_filenames = {} for filename in files: tag = filename.rsplit(".png", 1)[0] tree_filenames[tag] = filename return tree_filenames
def test_actually_parallel(self): results = subprocessing.parallel_function(dummy, [[0]] * 2) assert len(set(results)) == self.config_cpus # since it uses a pool, if we try with more we'll reuse those some procs results = subprocessing.parallel_function(os.getpid, [[]] * 8) assert len(set(results)) == self.config_cpus # but check it did run 8 times assert len(results) == 8
def test_results(self): # test single-arg functions res = subprocessing.parallel_function(len, [["a"], ["aa"], ["aaa"]]) assert res == [1, 2, 3] # test multi-arg functions res = subprocessing.parallel_function(zip, [["%d" % i, "b"] for i in range(3)]) assert [list(i) for i in res] == [[(str(i), "b")] for i in range(3)]
def test_local_funcs(self): # this test is mostly to let us know if something changes in pool/pickle def local(value): return value + 1 # ensure the function works as expected when called directly assert local(1) == 2 # check if it still fails within a parallel pool with self.assertRaisesRegex(AttributeError, "Can't pickle local object"): subprocessing.parallel_function(local, [[i] for i in range(3)])
def run_clusterblast_processes(options: ConfigType) -> None: """ Run blast in parallel, creates `options.cpu` number of files with the name format "inputN.fasta" Arguments: options: antismash Config Returns: None """ database = _get_datafile_path('subclusterprots.fasta') # set the first arg to always be database partial = functools.partial(_run_blast_helper, database) # run in parallel subprocessing.parallel_function(partial, [[i] for i in range(options.cpus)], cpus=options.cpus)
def test_errors_propagated(self): with self.assertRaisesRegex(ValueError, "Lucky number"): subprocessing.parallel_function(dummy, [[i] for i in range(5)])
def test_cpu_param(self): results = subprocessing.parallel_function(dummy, [[]] * 4, cpus=4) assert len(set(results)) == 4 assert len(results) == 4
def test_local_funcs(self): def local(value): return value + 1 # this test is mostly to let us know if something changes in pool/pickle with self.assertRaisesRegex(AttributeError, "Can't pickle local object"): subprocessing.parallel_function(local, [[i] for i in range(3)])