def make_msa(seqbuddy, aligner, trimal=()):
"""
Create a multiple sequence alignment
:param seqbuddy: SeqBuddy object
:param aligner: path to alignment program
:param trimal: List of TrimAl thresholds to try
:return: AlignBuddy object
"""
trimal = trimal if trimal else ["clean"]
if len(seqbuddy) == 1:
alignment = Alb.AlignBuddy(str(seqbuddy))
else:
alignment = Alb.generate_msa(Sb.make_copy(seqbuddy), aligner, quiet=True)
ave_seq_length = Sb.ave_seq_length(seqbuddy)
for threshold in trimal:
align_copy = Alb.trimal(Alb.make_copy(alignment), threshold=threshold)
cleaned_seqs = Sb.clean_seq(Sb.SeqBuddy(str(align_copy)))
cleaned_seqs = Sb.delete_small(cleaned_seqs, 1)
# Structured this way for unit test purposes
if len(alignment.records()) != len(cleaned_seqs):
continue
elif Sb.ave_seq_length(cleaned_seqs) / ave_seq_length < 0.5:
continue
else:
alignment = align_copy
break
return alignment
def get_data(self, data):
if data == "cteno_panxs":
return Sb.make_copy(self._cteno_panxs)
elif data == "cteno_panxs_aln":
return Alb.make_copy(self._cteno_panxs_aln)
elif data == "cteno_ids":
return deepcopy(self._cteno_ids)
elif data == "cteno_sim_scores":
return deepcopy(self._cteno_sim_scores)
elif data == "ss2_dfs":
psi_pred_ss2_dfs = Sb.OrderedDict()
for rec in cteno_panxs.records:
path = os.path.join(self.resource_path, "psi_pred",
"%s.ss2" % rec.id)
psi_pred_ss2_dfs[rec.id] = pd.read_csv(path,
comment="#",
header=None,
delim_whitespace=True)
psi_pred_ss2_dfs[rec.id].columns = [
"indx", "aa", "ss", "coil_prob", "helix_prob", "sheet_prob"
]
return psi_pred_ss2_dfs
elif data == "ss2_paths":
psi_pred_ss2 = Sb.OrderedDict()
for rec in cteno_panxs.records:
psi_pred_ss2[rec.id] = os.path.join(self.resource_path,
"psi_pred",
"%s.ss2" % rec.id)
return psi_pred_ss2
else:
raise AttributeError("Unknown data type: %s" % data)
ref_name = in_args.reference.split(os.sep)[-1]
ref_name = os.path.splitext(ref_name)[0]
if not os.path.isfile("%s%s.gb" % (ref_dir, ref_name)):
shutil.copy(in_args.reference, "%s%s.gb" % (ref_dir, ref_name))
if not os.path.isfile("%s%s_pep.gb" % (ref_dir, ref_name)):
sb_pep = Sb.translate_cds(Sb.make_copy(seqbuddy))
sb_pep.write("%s%s_pep.gb" % (ref_dir, ref_name))
if not os.path.isfile("%s%s_rna.gb" % (ref_dir, ref_name)):
sb_rna = Sb.dna2rna(Sb.make_copy(seqbuddy))
sb_rna.write("%s%s_rna.gb" % (ref_dir, ref_name))
if not os.path.isfile("%s%s_aln.gb" % (ref_dir, ref_name)):
alignbuddy = Alb.generate_msa(Sb.make_copy(seqbuddy), "mafft")
alignbuddy.write("%s%s_aln.gb" % (ref_dir, ref_name))
else:
alignbuddy = Alb.AlignBuddy("%s%s_aln.gb" % (ref_dir, ref_name))
if not os.path.isfile("%s%s_pep_aln.gb" % (ref_dir, ref_name)):
alb_pep = Alb.translate_cds(Alb.make_copy(alignbuddy))
alb_pep.write("%s%s_pep_aln.gb" % (ref_dir, ref_name))
if not os.path.isfile("%s%s_rna_aln.gb" % (ref_dir, ref_name)):
alb_rna = Alb.dna2rna(Alb.make_copy(alignbuddy))
alb_rna.write("%s%s_rna_aln.gb" % (ref_dir, ref_name))
if not os.path.isfile("%s%s_tree.nwk" % (ref_dir, ref_name)):
phylobuddy = Pb.generate_tree(Alb.make_copy(alignbuddy), "fasttree")
phylobuddy.write("%s%s_tree.nwk" % (ref_dir, ref_name))
def start(self):
self.split_time = time.time()
self.start_time = time.time()
self.heartbeat.start()
self.worker_file = os.path.join(self.working_dir,
"Worker_%s" % self.heartbeat.id)
with open(self.worker_file, "w") as ofile:
ofile.write("To terminate this Worker, simply delete this file.")
self.data_file = os.path.join(self.working_dir,
".Worker_%s.dat" % self.heartbeat.id)
open(self.data_file, "w").close()
helpers.dummy_func()
self.last_heartbeat_from_master = time.time()
self.printer.write("Starting Worker_%s" % self.heartbeat.id)
self.printer.new_line(1)
idle_countdown = 1
while os.path.isfile(self.worker_file):
idle = round(100 * self.idle / (self.idle + self.running), 2)
if not idle_countdown:
self.printer.write("Idle %s%%" % idle)
idle_countdown = 5
# Make sure there are some masters still kicking around
self.check_masters(idle)
# Check for and clean up dead threads and orphaned jobs every twentieth(ish) time through
rand_check = random()
if rand_check > 0.95:
self.clean_dead_threads()
# Fetch a job from the queue
data = self.fetch_queue_job()
if data:
full_name, psipred_dir, align_m, align_p, trimal, gap_open, gap_extend = data
subjob_num, num_subjobs, id_hash = [1, 1, full_name] if len(full_name.split("_")) == 1 \
else full_name.split("_")
subjob_num = int(subjob_num)
num_subjobs = int(num_subjobs)
self.printer.write("Running %s" % full_name)
else:
time.sleep(
random() * self.idle_workers()
) # Pause for some time relative to num idle workers
idle_countdown -= 1
self.idle += time.time() - self.split_time
self.split_time = time.time()
continue
try:
idle_countdown = 1
seqbuddy = Sb.SeqBuddy("%s/%s.seqs" % (self.output, id_hash),
in_format="fasta")
# Prepare alignment
if len(seqbuddy) == 1:
raise ValueError("Queued job of size 1 encountered: %s" %
id_hash)
else:
if num_subjobs == 1:
self.printer.write("Creating MSA (%s seqs)" %
len(seqbuddy))
alignment = Alb.generate_msa(Sb.make_copy(seqbuddy),
align_m,
params=align_p,
quiet=True)
else:
self.printer.write("Reading MSA (%s seqs)" %
len(seqbuddy))
alignment = Alb.AlignBuddy(
os.path.join(self.output, "%s.aln" % id_hash))
# Prepare psipred dataframes
psipred_dfs = self.prepare_psipred_dfs(seqbuddy, psipred_dir)
if num_subjobs == 1: # This is starting a full job from scratch, not a sub-job
# Need to specify what columns the PsiPred files map to now that there are gaps.
psipred_dfs = rdmcl.update_psipred(alignment, psipred_dfs,
"msa")
# TrimAl
self.printer.write("Trimal (%s seqs)" % len(seqbuddy))
alignment = rdmcl.trimal(seqbuddy, trimal, alignment)
with helpers.ExclusiveConnect(os.path.join(
self.output, "write.lock"),
max_lock=0):
# Place these write commands in ExclusiveConnect to ensure a writing lock
if not os.path.isfile(
os.path.join(self.output, "%s.aln" % id_hash)):
alignment.write(os.path.join(
self.output, "%s.aln" % id_hash),
out_format="fasta")
# Re-update PsiPred files now that some columns, possibly including non-gap characters, are removed
self.printer.write("Updating %s psipred dataframes" %
len(seqbuddy))
psipred_dfs = rdmcl.update_psipred(alignment, psipred_dfs,
"trimal")
# Prepare all-by-all list
self.printer.write("Preparing all-by-all data")
data_len, data = rdmcl.prepare_all_by_all(
seqbuddy, psipred_dfs, self.cpus)
if num_subjobs == 1 and data_len > self.cpus * self.job_size_coff:
data_len, data, subjob_num, num_subjobs = self.spawn_subjobs(
id_hash, data, psipred_dfs, gap_open, gap_extend)
elif subjob_num > 1:
data_len, data = self.load_subjob(id_hash, subjob_num,
num_subjobs, psipred_dfs)
# Launch multicore
self.printer.write("Running all-by-all data (%s comparisons)" %
data_len)
with open(self.data_file, "w") as ofile:
ofile.write("seq1,seq2,subsmat,psi")
br.run_multicore_function(data,
rdmcl.mc_score_sequences,
quiet=True,
max_processes=self.cpus,
func_args=[
alignment, gap_open, gap_extend,
self.data_file
])
self.printer.write("Processing final results")
self.process_final_results(id_hash, subjob_num, num_subjobs)
self.running += time.time() - self.split_time
self.split_time = time.time()
except (OSError, FileNotFoundError, br.GuessError,
ValueError) as err:
if num_subjobs == 1:
self.terminate(
"something wrong with primary cluster %s\n%s" %
(full_name, err))
else:
with helpers.ExclusiveConnect(self.wrkdb_path) as cursor:
cursor.execute("DELETE FROM processing WHERE hash=?",
(full_name, ))
continue
# Broken out of while loop, clean up and terminate worker
if os.path.isfile(self.data_file):
os.remove(self.data_file)
self.terminate("deleted check file")
outdir = os.path.abspath(in_args.out_dir)
if not os.path.exists(outdir):
print("Output directory not found, creating it...\n%s" % outdir, file=sys.stderr)
os.mkdir(outdir)
for block in blocks:
path = os.path.abspath(block.split("\n")[0])
file_name = path.split("/")[-1].split(".")[:-1]
file_name = "_".join(file_name)
file_name = "_".join(file_name.split(" "))
new_dir = os.path.abspath("%s/%s" % (outdir, file_name))
if not os.path.exists(new_dir):
os.mkdir(new_dir)
alignbuddy = Alb.AlignBuddy(path)
with open("%s/%s.phy" % (new_dir, file_name), 'w') as ofile:
ofile.write(screw_formats_align(alignbuddy))
# partitionfinder is super fussy about the PHYLIP format used. There needs to be exactly 10 characters in the IDs,
# with at least 2 spaces before the start of the actual sequence. So annoying.
Alb.hash_ids(alignbuddy, hash_length=8)
with open("%s/%s_hashed.phy" % (new_dir, file_name), "w") as ofile:
ofile.write(screw_formats_align(alignbuddy))
# Make cfg file
seq_ranges = block.split("\n")[1:]
if seq_ranges[-1] == "":
seq_ranges = seq_ranges[:-1]
alphabet = Alb.guess_alphabet(alignbuddy)
def test_make_msa(hf, monkeypatch):
seqbuddy = hf.get_data("cteno_panxs")
seqbuddy.records = seqbuddy.records[:2]
alb_obj = group_by_cluster.make_msa(seqbuddy, "clustalo")
assert type(alb_obj) == Alb.AlignBuddy
assert str(alb_obj) == """\
>Bab-PanxαA Be_abyssicola|m.8 and m.21|ML036514|937+ 2.
MLLLGSLGTIKNLSIFKDLSLDDWLDQMNRTFMFLLLCFMGTIVAVSQYTGKNISCNGFE
KFSDDFSQDYCWTQGLYTIKEAYDLPESQIPYPGIIPENVPACREHSLKNGGKIICPPPE
EIKPLTRARHLWYQWIPFYFWVIAPVFYLPYMFVKRMGLDRMKPLLKIMSDYYHCTTETP
SEEIIVKCADWVYNSIVDRLSEGSSWTSWRNRHGLGLAVLFSKLMYLGGSILVMMVTTLM
FQVGDFKTYGIEWLKQFPSDENYTTSVKHKLFPKMVACEIKRWGPSGLEEENGMCVLAPN
VIYQYIFLIMWFALAITICTNFFNIFFWVFKLTATRYTYSKLVATGHFSHKHPGWKFMYY
RIGTSGRVLLNIVAQNTNPIIFGAIMEKLTPSVIKHLRIGHVPGEYLTDPA
>Bab-PanxαB Be_abyssicola|m.19|ML47742|1063 2.
--MLDILSKFKGVTPFKGITIDDGWDQLNRSFMFVLLVVMGTTVTVRQYTGSVISCDGFK
KFGSTFAEDYCWTQGLYTVLEGYDQPSYNIPYPGLLPDELPACTPVKLKDGTRLKCPDAD
QLMSPTRISHLWYQWVPFYFWLAAAAFFMPYLLYKNFGMGDIKPLVRLLHNPVESDQ--E
LKKMTDKAATWLFYKFDLYMSEQSLVASLTRKHGLGLSMVFVKILYAAVSFCCFILTAEM
FSIGDFKTYGSKWIKKMRYEDTLATEEKDKLFPKMVACEVKRWGASGIEEEQGMCVLAPN
VINQYLFLILWFCLVFVMICNIVSIFVSLIKLLFTYGSYRRLLST-AFLRDDSAIKHMYF
NVGSSGRLILHVLANNTAPRVFEDILLTLAPKLIQRKLRGNGKAV------
"""
seqbuddy.records = [seqbuddy.records[0]]
alb_obj = group_by_cluster.make_msa(seqbuddy, "clustalo")
assert type(alb_obj) == Alb.AlignBuddy
assert str(alb_obj) == """\
>Bab-PanxαA Be_abyssicola|m.8 and m.21|ML036514|937+ 2.
MLLLGSLGTIKNLSIFKDLSLDDWLDQMNRTFMFLLLCFMGTIVAVSQYTGKNISCNGFE
KFSDDFSQDYCWTQGLYTIKEAYDLPESQIPYPGIIPENVPACREHSLKNGGKIICPPPE
EIKPLTRARHLWYQWIPFYFWVIAPVFYLPYMFVKRMGLDRMKPLLKIMSDYYHCTTETP
SEEIIVKCADWVYNSIVDRLSEGSSWTSWRNRHGLGLAVLFSKLMYLGGSILVMMVTTLM
FQVGDFKTYGIEWLKQFPSDENYTTSVKHKLFPKMVACEIKRWGPSGLEEENGMCVLAPN
VIYQYIFLIMWFALAITICTNFFNIFFWVFKLTATRYTYSKLVATGHFSHKHPGWKFMYY
RIGTSGRVLLNIVAQNTNPIIFGAIMEKLTPSVIKHLRIGHVPGEYLTDPA
"""
# Don't modify if any sequence is reduced to nothing
align = Alb.AlignBuddy("""\
>A
MSTGTC-------
>B
M---TC-------
>C
M---TC---AILP
>D
-STP---YWAILP
""", in_format="fasta")
seqbuddy = Sb.SeqBuddy(Alb.make_copy(align).records(), in_format="fasta")
seqbuddy = Sb.clean_seq(seqbuddy)
monkeypatch.setattr(Alb, "generate_msa", lambda *_, **__: align)
alb_obj = group_by_cluster.make_msa(seqbuddy, "clustalo", trimal=[0.3])
assert str(alb_obj) == str(align)
# Don't modify if average sequence length is reduced by more than half
align = Alb.AlignBuddy("""\
>A
MSTGTC-------
>B
M---TC-------
>C
M---TC---AILP
>D
-STPTC-YWAILP
""", in_format="fasta")
seqbuddy = Sb.SeqBuddy(Alb.make_copy(align).records(), in_format="fasta")
seqbuddy = Sb.clean_seq(seqbuddy)
monkeypatch.setattr(Alb, "generate_msa", lambda *_, **__: align)
alb_obj = group_by_cluster.make_msa(seqbuddy, "clustalo", trimal=[0.3])
assert str(alb_obj) == str(align)
# Remove some gaps
alb_obj = group_by_cluster.make_msa(seqbuddy, "clustalo", trimal=[0.3, 0.55])
assert str(alb_obj) == """\
def test_make_msa(hf, monkeypatch):
seqbuddy = hf.get_data("cteno_panxs")
seqbuddy.records = seqbuddy.records[:2]
alb_obj = group_by_cluster.make_msa(seqbuddy, "clustalo")
assert type(alb_obj) == Alb.AlignBuddy
assert str(alb_obj) == """\
>Bab-PanxαA Be_abyssicola|m.8 and m.21|ML036514|937+ 2.
MLLLGSLGTIKNLSIFKDLSLDDWLDQMNRTFMFLLLCFMGTIVAVSQYTGKNISCNGFE
KFSDDFSQDYCWTQGLYTIKEAYDLPESQIPYPGIIPENVPACREHSLKNGGKIICPPPE
EIKPLTRARHLWYQWIPFYFWVIAPVFYLPYMFVKRMGLDRMKPLLKIMSDYYHCTTETP
SEEIIVKCADWVYNSIVDRLSEGSSWTSWRNRHGLGLAVLFSKLMYLGGSILVMMVTTLM
FQVGDFKTYGIEWLKQFPSDENYTTSVKHKLFPKMVACEIKRWGPSGLEEENGMCVLAPN
VIYQYIFLIMWFALAITICTNFFNIFFWVFKLTATRYTYSKLVATGHFSHKHPGWKFMYY
RIGTSGRVLLNIVAQNTNPIIFGAIMEKLTPSVIKHLRIGHVPGEYLTDPA
>Bab-PanxαB Be_abyssicola|m.19|ML47742|1063 2.
--MLDILSKFKGVTPFKGITIDDGWDQLNRSFMFVLLVVMGTTVTVRQYTGSVISCDGFK
KFGSTFAEDYCWTQGLYTVLEGYDQPSYNIPYPGLLPDELPACTPVKLKDGTRLKCPDAD
QLMSPTRISHLWYQWVPFYFWLAAAAFFMPYLLYKNFGMGDIKPLVRLLHNPVESDQ--E
LKKMTDKAATWLFYKFDLYMSEQSLVASLTRKHGLGLSMVFVKILYAAVSFCCFILTAEM
FSIGDFKTYGSKWIKKMRYEDTLATEEKDKLFPKMVACEVKRWGASGIEEEQGMCVLAPN
VINQYLFLILWFCLVFVMICNIVSIFVSLIKLLFTYGSYRRLLST-AFLRDDSAIKHMYF
NVGSSGRLILHVLANNTAPRVFEDILLTLAPKLIQRKLRGNGKAV------
"""
seqbuddy.records = [seqbuddy.records[0]]
alb_obj = group_by_cluster.make_msa(seqbuddy, "clustalo")
assert type(alb_obj) == Alb.AlignBuddy
assert str(alb_obj) == """\
>Bab-PanxαA Be_abyssicola|m.8 and m.21|ML036514|937+ 2.
MLLLGSLGTIKNLSIFKDLSLDDWLDQMNRTFMFLLLCFMGTIVAVSQYTGKNISCNGFE
KFSDDFSQDYCWTQGLYTIKEAYDLPESQIPYPGIIPENVPACREHSLKNGGKIICPPPE
EIKPLTRARHLWYQWIPFYFWVIAPVFYLPYMFVKRMGLDRMKPLLKIMSDYYHCTTETP
SEEIIVKCADWVYNSIVDRLSEGSSWTSWRNRHGLGLAVLFSKLMYLGGSILVMMVTTLM
FQVGDFKTYGIEWLKQFPSDENYTTSVKHKLFPKMVACEIKRWGPSGLEEENGMCVLAPN
VIYQYIFLIMWFALAITICTNFFNIFFWVFKLTATRYTYSKLVATGHFSHKHPGWKFMYY
RIGTSGRVLLNIVAQNTNPIIFGAIMEKLTPSVIKHLRIGHVPGEYLTDPA
"""
# Don't modify if any sequence is reduced to nothing
align = Alb.AlignBuddy("""\
>A
MSTGTC-------
>B
M---TC-------
>C
M---TC---AILP
>D
-STP---YWAILP
""",
in_format="fasta")
seqbuddy = Sb.SeqBuddy(Alb.make_copy(align).records(), in_format="fasta")
seqbuddy = Sb.clean_seq(seqbuddy)
monkeypatch.setattr(Alb, "generate_msa", lambda *_, **__: align)
alb_obj = group_by_cluster.make_msa(seqbuddy, "clustalo", trimal=[0.3])
assert str(alb_obj) == str(align)
# Don't modify if average sequence length is reduced by more than half
align = Alb.AlignBuddy("""\
>A
MSTGTC-------
>B
M---TC-------
>C
M---TC---AILP
>D
-STPTC-YWAILP
""",
in_format="fasta")
seqbuddy = Sb.SeqBuddy(Alb.make_copy(align).records(), in_format="fasta")
seqbuddy = Sb.clean_seq(seqbuddy)
monkeypatch.setattr(Alb, "generate_msa", lambda *_, **__: align)
alb_obj = group_by_cluster.make_msa(seqbuddy, "clustalo", trimal=[0.3])
assert str(alb_obj) == str(align)
# Remove some gaps
alb_obj = group_by_cluster.make_msa(seqbuddy,
"clustalo",
trimal=[0.3, 0.55])
assert str(alb_obj) == """\
Cfu 6
Dgl 9
Edu 9
Hca 8
Hru 5
Hvu 14
Lcr 12
Lla 3
Mle 12
Oma 4
Pba 7
Tin 6
Vpa 7
'''
cteno_panxs = Sb.SeqBuddy("%s%sCteno_pannexins.fa" % (RESOURCE_PATH, SEP))
cteno_panxs_aln = Alb.AlignBuddy("%s%sCteno_pannexins_aln.fa" %
(RESOURCE_PATH, SEP))
ids = sorted([rec.id for rec in cteno_panxs.records])
sim_scores = pd.read_csv("%sCteno_pannexins_sim.scores" % RESOURCE_PATH,
index_col=False,
header=None)
sim_scores.columns = ["seq1", "seq2", "subsmat", "psi", "raw_score", "score"]
# ################################# - Helper class - ################################## #
class HelperMethods(object):
def __init__(self):
self.sep = SEP
self.resource_path = RESOURCE_PATH
self._cteno_panxs = cteno_panxs
self._cteno_panxs_aln = cteno_panxs_aln
self._cteno_ids = ids
def main():
in_args = argparse_init()
mode = in_args.mode.lower()
mode = "seqs" if "sequences".startswith(mode) else mode
mode = "aln" if "alignment".startswith(mode) else mode
mode = "con" if "consensus".startswith(mode) else mode
mode = "list" if "list".startswith(mode) else mode
if mode not in ["seqs", "aln", "con", "list"]:
Sb.br._stderr('Unrecognized mode, please select from ["seqs", "aln", "con", "list"].\n')
sys.exit()
if in_args.groups:
in_args.groups = [x.lower() for x in in_args.groups[0]]
in_args.groups = "^%s$" % "$|^".join(in_args.groups)
cluster_file = prepare_clusters(in_args.clusters, hierarchy=True)
seqbuddy = Sb.SeqBuddy(in_args.sequence_file)
output = OrderedDict()
for rank, node in cluster_file.items():
rank = rank.split()[0]
if in_args.groups:
if not re.search(in_args.groups, rank):
continue
if in_args.min_size:
if len(node) < in_args.min_size:
continue
if in_args.max_size:
if len(node) > in_args.max_size:
continue
if in_args.strip_taxa:
node = [re.sub("^.*?\-", "", x) for x in node]
ids = "^%s$" % "$|^".join(node)
subset = Sb.pull_recs(Sb.make_copy(seqbuddy), ids)
subset = Sb.order_ids(subset)
rank_output = ""
if mode == "list":
rank_output += rank
for rec in subset.records:
rec.description = re.sub("^%s" % rec.id, "", rec.description)
rank_output += "\n%s %s" % (rec.id, rec.description)
rank_output += "\n"
elif mode == "seqs":
for rec in subset.records:
rec.description = "%s %s" % (rank, rec.description)
rank_output += str(subset)
elif mode in ["aln", "con"]:
try:
rank_output = make_msa(subset, in_args.aligner, in_args.trimal)
except (SystemError, AttributeError) as err:
print(err)
sys.exit()
rank_output.out_format = "phylip-relaxed"
if mode == "con":
rec = Alb.consensus_sequence(rank_output).records()[0]
rec.id = rank
rec.name = rank
rec.description = ""
rank_output.out_format = "fasta"
output[rank] = str(rank_output)
if not in_args.write:
print("\n".join(data for rank, data in output.items()).strip())
else:
outdir = os.path.abspath(in_args.write)
os.makedirs(outdir, exist_ok=True)
extension = ".%s" % seqbuddy.out_format[:3] if mode == "seq" \
else ".txt" if mode == "list" \
else ".phy" if mode == "aln" \
else ".fa"
for rank, data in output.items():
with open(os.path.join(outdir, rank + extension), "w") as ofile:
ofile.write(data)
def start(self):
self.split_time = time.time()
self.start_time = time.time()
self.heartbeat.start()
self.worker_file = os.path.join(self.working_dir, "Worker_%s" % self.heartbeat.id)
with open(self.worker_file, "w") as ofile:
ofile.write("To terminate this Worker, simply delete this file.")
self.data_file = os.path.join(self.working_dir, ".Worker_%s.dat" % self.heartbeat.id)
open(self.data_file, "w").close()
helpers.dummy_func()
self.last_heartbeat_from_master = time.time()
self.printer.write("Starting Worker_%s" % self.heartbeat.id)
self.printer.new_line(1)
idle_countdown = 1
while os.path.isfile(self.worker_file):
idle = round(100 * self.idle / (self.idle + self.running), 2)
if not idle_countdown:
self.printer.write("Idle %s%%" % idle)
idle_countdown = 5
# Make sure there are some masters still kicking around
self.check_masters(idle)
# Check for and clean up dead threads and orphaned jobs every twentieth(ish) time through
rand_check = random()
if rand_check > 0.95:
self.clean_dead_threads()
# Fetch a job from the queue
data = self.fetch_queue_job()
if data:
full_name, psipred_dir, align_m, align_p, trimal, gap_open, gap_extend = data
subjob_num, num_subjobs, id_hash = [1, 1, full_name] if len(full_name.split("_")) == 1 \
else full_name.split("_")
subjob_num = int(subjob_num)
num_subjobs = int(num_subjobs)
self.printer.write("Running %s" % full_name)
else:
time.sleep(random() * self.idle_workers()) # Pause for some time relative to num idle workers
idle_countdown -= 1
self.idle += time.time() - self.split_time
self.split_time = time.time()
continue
try:
idle_countdown = 1
seqbuddy = Sb.SeqBuddy("%s/%s.seqs" % (self.output, id_hash), in_format="fasta")
# Prepare alignment
if len(seqbuddy) == 1:
raise ValueError("Queued job of size 1 encountered: %s" % id_hash)
else:
if num_subjobs == 1:
self.printer.write("Creating MSA (%s seqs)" % len(seqbuddy))
alignment = Alb.generate_msa(Sb.make_copy(seqbuddy), align_m,
params=align_p, quiet=True)
else:
self.printer.write("Reading MSA (%s seqs)" % len(seqbuddy))
alignment = Alb.AlignBuddy(os.path.join(self.output, "%s.aln" % id_hash))
# Prepare psipred dataframes
psipred_dfs = self.prepare_psipred_dfs(seqbuddy, psipred_dir)
if num_subjobs == 1: # This is starting a full job from scratch, not a sub-job
# Need to specify what columns the PsiPred files map to now that there are gaps.
psipred_dfs = rdmcl.update_psipred(alignment, psipred_dfs, "msa")
# TrimAl
self.printer.write("Trimal (%s seqs)" % len(seqbuddy))
alignment = rdmcl.trimal(seqbuddy, trimal, alignment)
with helpers.ExclusiveConnect(os.path.join(self.output, "write.lock"), max_lock=0):
# Place these write commands in ExclusiveConnect to ensure a writing lock
if not os.path.isfile(os.path.join(self.output, "%s.aln" % id_hash)):
alignment.write(os.path.join(self.output, "%s.aln" % id_hash), out_format="fasta")
# Re-update PsiPred files now that some columns, possibly including non-gap characters, are removed
self.printer.write("Updating %s psipred dataframes" % len(seqbuddy))
psipred_dfs = rdmcl.update_psipred(alignment, psipred_dfs, "trimal")
# Prepare all-by-all list
self.printer.write("Preparing all-by-all data")
data_len, data = rdmcl.prepare_all_by_all(seqbuddy, psipred_dfs, self.cpus)
if num_subjobs == 1 and data_len > self.cpus * self.job_size_coff:
data_len, data, subjob_num, num_subjobs = self.spawn_subjobs(id_hash, data, psipred_dfs,
gap_open, gap_extend)
elif subjob_num > 1:
data_len, data = self.load_subjob(id_hash, subjob_num, num_subjobs, psipred_dfs)
# Launch multicore
self.printer.write("Running all-by-all data (%s comparisons)" % data_len)
with open(self.data_file, "w") as ofile:
ofile.write("seq1,seq2,subsmat,psi")
br.run_multicore_function(data, rdmcl.mc_score_sequences, quiet=True, max_processes=self.cpus,
func_args=[alignment, gap_open, gap_extend, self.data_file])
self.printer.write("Processing final results")
self.process_final_results(id_hash, subjob_num, num_subjobs)
self.running += time.time() - self.split_time
self.split_time = time.time()
except (OSError, FileNotFoundError, br.GuessError, ValueError) as err:
if num_subjobs == 1:
self.terminate("something wrong with primary cluster %s\n%s" % (full_name, err))
else:
with helpers.ExclusiveConnect(self.wrkdb_path) as cursor:
cursor.execute("DELETE FROM processing WHERE hash=?", (full_name,))
continue
# Broken out of while loop, clean up and terminate worker
if os.path.isfile(self.data_file):
os.remove(self.data_file)
self.terminate("deleted check file")
if not os.path.isfile("%s%s_pep.gb" % (ref_dir, ref_name)):
print(" -> Creating protein file")
sb_pep = Sb.translate_cds(Sb.make_copy(seqbuddy), quiet=True)
sb_pep.write("%s%s_pep.gb" % (ref_dir, ref_name))
del sb_pep
if not os.path.isfile("%s%s_rna.gb" % (ref_dir, ref_name)):
print(" -> Creating RNA file")
sb_rna = Sb.dna2rna(Sb.make_copy(seqbuddy))
sb_rna.write("%s%s_rna.gb" % (ref_dir, ref_name))
del sb_rna
if not os.path.isfile("%s%s_aln.gb" % (ref_dir, ref_name)):
print(" -> Creating alignment file")
alignbuddy = Alb.faux_alignment(Sb.make_copy(seqbuddy), r_seed=12345)
alignbuddy.write("%s%s_aln.gb" % (ref_dir, ref_name))
else:
alignbuddy = Alb.AlignBuddy("%s%s_aln.gb" % (ref_dir, ref_name))
if not os.path.isfile("%s%s_pep_aln.gb" % (ref_dir, ref_name)):
print(" -> Creating protein alignment file")
alb_pep = Alb.faux_alignment(
Sb.SeqBuddy("%s%s_pep.gb" % (ref_dir, ref_name)))
alb_pep.write("%s%s_pep_aln.gb" % (ref_dir, ref_name))
del alb_pep
if not os.path.isfile("%s%s_rna_aln.gb" % (ref_dir, ref_name)):
print(" -> Creating RNA alignment file")
alb_rna = Alb.dna2rna(Alb.make_copy(alignbuddy))
alb_rna.write("%s%s_rna_aln.gb" % (ref_dir, ref_name))