def generate_trees(smcogs_dir: str, genes_within_clusters: List[CDSFeature],
nrpspks_genes: List[CDSFeature]) -> Dict[str, str]:
""" smCOG phylogenetic tree construction """
pks_nrps_cds_names = set(feature.get_name() for feature in nrpspks_genes)
logging.info("Calculating and drawing phylogenetic trees of cluster genes "
"with smCOG members")
cds_features = []
for cds in genes_within_clusters:
cds_name = cds.get_name()
if cds_name in pks_nrps_cds_names:
continue
if not cds.gene_functions.get_by_tool("smcogs"):
continue
cds_features.append(cds)
with TemporaryDirectory(change=True):
args = []
for index, cds in enumerate(cds_features):
smcog = cds.gene_functions.get_by_tool("smcogs")[0].description.split(":")[0]
args.append([cds, index, smcog, smcogs_dir])
subprocessing.parallel_function(smcog_tree_analysis, args)
files = glob.glob("*.png")
tree_filenames = {}
for filename in files:
tag = filename.rsplit(".png", 1)[0]
tree_filenames[tag] = filename
return tree_filenames
def test_timeout(self):
start = time.time()
timeout = 1
with self.assertRaisesRegex(RuntimeError, "Timeout in parallel .*"):
subprocessing.parallel_function(time.sleep, [[5]]*2, timeout=timeout)
elapsed = time.time() - start
assert elapsed < 1.5
def generate_trees(smcogs_dir: str, hmm_results: Dict[str, List[HSP]],
genes_within_clusters: List[CDSFeature],
nrpspks_genes: List[CDSFeature]) -> Dict[str, str]:
""" smCOG phylogenetic tree construction """
pks_nrps_gene_names = set(
[feature.get_name() for feature in nrpspks_genes])
logging.info("Calculating and drawing phylogenetic trees of cluster genes "
"with smCOG members")
with TemporaryDirectory(change=True):
cds_features = []
for cds in genes_within_clusters:
gene_id = cds.get_name()
if gene_id not in pks_nrps_gene_names and hmm_results.get(gene_id):
cds_features.append(cds)
args = []
for index, cds in enumerate(cds_features):
smcog = hmm_results[cds.get_name()][0].hit_id.split(":")[0]
args.append([cds, index, smcog, smcogs_dir])
subprocessing.parallel_function(smcog_tree_analysis, args)
files = glob.glob("*.png")
tree_filenames = {}
for filename in files:
tag = filename.rsplit(".png", 1)[0]
tree_filenames[tag] = filename
return tree_filenames
def test_actually_parallel(self):
results = subprocessing.parallel_function(dummy, [[0]] * 2)
assert len(set(results)) == self.config_cpus
# since it uses a pool, if we try with more we'll reuse those some procs
results = subprocessing.parallel_function(os.getpid, [[]] * 8)
assert len(set(results)) == self.config_cpus
# but check it did run 8 times
assert len(results) == 8
def test_results(self):
# test single-arg functions
res = subprocessing.parallel_function(len, [["a"], ["aa"], ["aaa"]])
assert res == [1, 2, 3]
# test multi-arg functions
res = subprocessing.parallel_function(zip, [["%d" % i, "b"] for i in range(3)])
assert [list(i) for i in res] == [[(str(i), "b")] for i in range(3)]
def test_local_funcs(self):
# this test is mostly to let us know if something changes in pool/pickle
def local(value):
return value + 1
# ensure the function works as expected when called directly
assert local(1) == 2
# check if it still fails within a parallel pool
with self.assertRaisesRegex(AttributeError, "Can't pickle local object"):
subprocessing.parallel_function(local, [[i] for i in range(3)])
def run_clusterblast_processes(options: ConfigType) -> None:
""" Run blast in parallel, creates `options.cpu` number of files with
the name format "inputN.fasta"
Arguments:
options: antismash Config
Returns:
None
"""
database = _get_datafile_path('subclusterprots.fasta')
# set the first arg to always be database
partial = functools.partial(_run_blast_helper, database)
# run in parallel
subprocessing.parallel_function(partial, [[i] for i in range(options.cpus)],
cpus=options.cpus)
def test_errors_propagated(self):
with self.assertRaisesRegex(ValueError, "Lucky number"):
subprocessing.parallel_function(dummy, [[i] for i in range(5)])
def test_cpu_param(self):
results = subprocessing.parallel_function(dummy, [[]] * 4, cpus=4)
assert len(set(results)) == 4
assert len(results) == 4
def test_local_funcs(self):
def local(value):
return value + 1
# this test is mostly to let us know if something changes in pool/pickle
with self.assertRaisesRegex(AttributeError, "Can't pickle local object"):
subprocessing.parallel_function(local, [[i] for i in range(3)])
