def buildInducedSubalignment(**kwargs):
alignmentColumnsPath = kwargs["alignmentColumnsPath"]
subalignmentPath = kwargs["subalignmentPath"]
inducedAlignPath = kwargs["outputFile"]
tempInducedAlignPath = os.path.join(
os.path.dirname(inducedAlignPath),
"temp_{}".format(os.path.basename(inducedAlignPath)))
alignColumns = []
with open(alignmentColumnsPath) as f:
insertIdxs = set(
[int(token) for token in f.readline().strip().split()])
for line in f:
tokens = set([int(token) for token in line.strip().split()])
alignColumns.append(tokens)
subsetAlign = sequenceutils.readFromFasta(subalignmentPath,
removeDashes=False)
inducedAlign = {taxon: ['-'] * len(alignColumns) for taxon in subsetAlign}
for idx, column in enumerate(alignColumns):
for taxon in subsetAlign:
for c in column:
letter = subsetAlign[taxon].seq[c]
if letter != '-':
letter = letter.lower() if c in insertIdxs else letter
assert inducedAlign[taxon][idx] == '-'
inducedAlign[taxon][idx] = letter
for s in inducedAlign:
inducedAlign[s] = sequenceutils.Sequence(s, "".join(inducedAlign[s]))
sequenceutils.writeFasta(inducedAlign, tempInducedAlignPath)
shutil.move(tempInducedAlignPath, inducedAlignPath)
def buildDecomposition(context, subsetsDir):
if not os.path.exists(subsetsDir):
os.makedirs(subsetsDir)
if context.unalignedSequences is None:
context.unalignedSequences = sequenceutils.readFromFasta(
context.sequencesPath, removeDashes=True)
if (Configs.decompositionStrategy == "random" or context.guideTree
== "random") and Configs.outputPath == context.outputFile:
context.subsetPaths = randomDecomposition(
subsetsDir, context.unalignedSequences,
Configs.decompositionMaxNumSubsets)
elif Configs.decompositionStrategy == "kmh":
Configs.log("Decomposing {} with KMH..".format(context.sequencesPath))
Configs.log("Targetting {} subsets..".format(
Configs.decompositionMaxNumSubsets))
context.subsetPaths = kmh.buildSubsetsKMH(context, subsetsDir)
else:
guideTreePath = initial_tree.buildInitialTree(context, subsetsDir,
context.guideTree)
Configs.log(
"Using target subset size of {}, and maximum number of subsets {}.."
.format(Configs.decompositionMaxSubsetSize,
Configs.decompositionMaxNumSubsets))
context.subsetPaths = treeutils.decomposeGuideTree(
subsetsDir, context.sequencesPath, guideTreePath,
Configs.decompositionMaxSubsetSize,
Configs.decompositionMaxNumSubsets)
def compressSubalignment(**kwargs):
subalignmentPath = kwargs["subalignmentPath"]
outputPath = kwargs["outputFile"]
tempOutputPath = os.path.join(
os.path.dirname(outputPath),
"temp_{}".format(os.path.basename(outputPath)))
subsetAlign = sequenceutils.readFromFasta(subalignmentPath,
removeDashes=False)
subsetLen = len(next(iter(subsetAlign.values())).seq)
numLetters = []
compressions = []
lastIdx = {s: -1 for s in subsetAlign}
for i in range(subsetLen):
notGaps = []
for s, v in subsetAlign.items():
if v.seq[i] != '-':
notGaps.append(s)
numLetters.append(len(notGaps))
dest = set()
for s in notGaps:
if lastIdx[s] != -1:
dest.add(lastIdx[s])
lastIdx[s] = i
compressions.append(dest)
with open(tempOutputPath, 'w') as textFile:
textFile.write("{}\n".format(" ".join([str(c) for c in numLetters])))
for dest in compressions:
textFile.write("{}\n".format(" ".join([str(c) for c in dest])))
shutil.move(tempOutputPath, outputPath)
def requestBackboneTasks(context):
if len(context.backbonePaths) > 0:
Configs.log("Using {} user-defined backbone files..".format(
len(context.backbonePaths)))
context.backbonePaths = context.backbonePaths
for path in context.backbonePaths:
context.backboneTaxa.update(sequenceutils.readFromFasta(path))
elif Configs.graphBuildMethod == "mafft":
Configs.log("Using {} MAFFT backbones..".format(Configs.mafftRuns))
requestMafftBackbones(context)
elif Configs.graphBuildMethod == "subsethmm":
Configs.log(
"Using {} HMM-extended subalignments as backbone files..".format(
len(context.subalignmentPaths)))
context.backbonePaths = context.subalignmentPaths
context.backboneExtend.update(context.backbonePaths)
elif Configs.graphBuildMethod == "initial":
Configs.log(
"Using the initial decomposition alignment as the single backbone.."
)
initialAlignPath = os.path.join(context.workingDir, "decomposition",
"initial_tree",
"initial_insert_align.txt")
context.backbonePaths = [initialAlignPath]
if not Configs.constrain and Configs.graphBuildMethod != "subsethmm":
context.backbonePaths.extend(context.subalignmentPaths)
def addAlignmentFileToGraph(context, alignedFile):
Configs.log("Feeding backbone {} to the graph..".format(alignedFile))
backboneAlign = sequenceutils.readFromFasta(alignedFile)
alignmentLength = len(next(iter(backboneAlign.values())).seq)
if alignedFile in context.backboneExtend:
extensionTasks = requestHmmExtensionTasks(context, backboneAlign,
alignedFile)
task.submitTasks(extensionTasks)
for extensionTask in task.asCompleted(extensionTasks):
backboneAlign.update(
sequenceutils.readFromStockholm(extensionTask.outputFile,
includeInsertions=True))
alignmap = backboneToAlignMap(context, backboneAlign, alignmentLength)
Configs.log(
"Constructed backbone alignment map from {}".format(alignedFile))
graph = context.graph
with graph.matrixLock:
for l in range(alignmentLength):
for a, avalue in alignmap[l].items():
for b, bvalue in alignmap[l].items():
if Configs.graphBuildRestrict:
asub, apos = graph.matSubPosMap[a]
bsub, bpos = graph.matSubPosMap[b]
if asub == bsub and apos != bpos:
continue
graph.matrix[a][b] = graph.matrix[a].get(
b, 0) + avalue * bvalue
Configs.log("Fed backbone {} to the graph.".format(alignedFile))
def decomposeGuideTree(subsetsDir, sequencesPath, guideTreePath, maxSubsetSize,
maxNumSubsets):
sequences = sequenceutils.readFromFasta(sequencesPath, removeDashes=False)
guideTree = dendropy.Tree.get(path=guideTreePath,
schema="newick",
preserve_underscores=True)
guideTree.collapse_basal_bifurcation()
for edge in guideTree.postorder_edge_iter():
if len(edge.head_node.child_edges()) > 0:
edge.childs = sum([e.childs for e in edge.head_node.child_edges()])
else:
edge.childs = 1
guideTree.childs = guideTree.seed_node.edge.childs
trees = decomposeTree(guideTree, maxSubsetSize, maxNumSubsets)
taxonSubsets = []
for tree in trees:
keep = [n.taxon.label for n in tree.leaf_nodes()]
taxonSubsets.append(keep)
subsetPaths = []
for n, subset in enumerate(taxonSubsets):
subsetPath = os.path.join(subsetsDir, "subset_{}.txt".format(n + 1))
subsetPaths.append(subsetPath)
sequenceutils.writeFasta(sequences, subsetPath, subset)
return subsetPaths
def buildHmmScores(hmmPaths, queriesPath, scoreFileHmmFileMap):
#tasks = [getHmmScores(hmmPath, queriesPath) for hmmPath in hmmPaths]
queries = sequenceutils.readFromFasta(queriesPath, removeDashes=True)
baseName = os.path.basename(queriesPath).split('.')[0]
dirName = os.path.join(os.path.dirname(queriesPath),
"chunks_{}".format(baseName))
if not os.path.exists(dirName):
os.makedirs(dirName)
chunkSize = 1000
taxa = list(queries.keys())
inputOutputs = []
for i in range(math.ceil(len(taxa) / chunkSize)):
chunk = taxa[i * chunkSize:min(len(taxa), (i + 1) * chunkSize)]
inputName = os.path.join(dirName,
"{}_chunk_{}.txt".format(baseName, i + 1))
sequenceutils.writeFasta(queries, inputName, chunk)
for hmmPath in hmmPaths:
outputName = os.path.join(
os.path.dirname(hmmPath),
"{}_chunk_{}_score.txt".format(baseName, i + 1))
inputOutputs.append((hmmPath, inputName, outputName))
scoreFileHmmFileMap[outputName] = hmmPath
tasks = [
getHmmScores(hmmPath, inputPath, outputPath)
for hmmPath, inputPath, outputPath in inputOutputs
]
return tasks
def hmmAlignQueries(hmmPath, queriesPath):
queries = sequenceutils.readFromFasta(queriesPath, removeDashes=True)
baseName = os.path.basename(queriesPath).split('.')[0]
dirName = os.path.join(os.path.dirname(queriesPath),
"chunks_{}".format(baseName))
if not os.path.exists(dirName):
os.makedirs(dirName)
chunkSize = 1000
taxa = list(queries.keys())
alignFiles = {}
for i in range(math.ceil(len(taxa) / chunkSize)):
chunk = taxa[i * chunkSize:min(len(taxa), (i + 1) * chunkSize)]
inputName = os.path.join(dirName,
"{}_chunk_{}.txt".format(baseName, i + 1))
outputName = os.path.join(
dirName, "{}_chunk_{}_aligned.txt".format(baseName, i + 1))
sequenceutils.writeFasta(queries, inputName, chunk)
alignFiles[inputName] = outputName
tasks = []
for inputPath, outputPath in alignFiles.items():
task = buildHmmAlignment(hmmPath, inputPath, outputPath)
tasks.append(task)
return tasks
def reassignTaxons(subsetsDir, subsetSeedPaths, sequences, unusedTaxa):
unusedPath = os.path.join(subsetsDir, "unassigned_sequences.txt")
sequenceutils.writeFasta(sequences, unusedPath, unusedTaxa)
hmmMap = {}
for subsetPath in subsetSeedPaths:
hmmDir = os.path.join(
os.path.dirname(subsetPath),
"hmm_{}".format(os.path.basename(subsetPath)).replace(".", "_"))
if not os.path.exists(hmmDir):
os.makedirs(hmmDir)
hmmMap[subsetPath] = os.path.join(hmmDir, "hmm_model.txt")
hmmTasks = hmmutils.buildHmms(hmmMap)
task.submitTasks(hmmTasks)
task.awaitTasks(hmmTasks)
hmmPaths = [t.outputFile for t in hmmTasks]
scoreFileHmmFileMap = {}
scoreTasks = hmmutils.buildHmmScores(hmmPaths, unusedPath,
scoreFileHmmFileMap)
task.submitTasks(scoreTasks)
bestScores = {}
taxonHmmMap = {}
for scoreTask in task.asCompleted(scoreTasks):
subsetScores = hmmutils.readSearchFile(scoreTask.outputFile)
for taxon, scores in subsetScores.items():
if scores[1] > bestScores.get(taxon, -float("inf")):
bestScores[taxon] = scores[1]
taxonHmmMap[taxon] = scoreFileHmmFileMap[scoreTask.outputFile]
subsetTaxons = {file: [] for file in hmmPaths}
for taxon, hmmPath in taxonHmmMap.items():
subsetTaxons[hmmPath].append(taxon)
for subsetPath, hmmPath in hmmMap.items():
subset = sequenceutils.readFromFasta(subsetPath)
for taxon in subset:
subsetTaxons[hmmPath].append(taxon)
subsetPaths = []
i = 1
for hmmPath, subset in subsetTaxons.items():
subsetPath = os.path.join(subsetsDir, "subset_{}.txt".format(i))
subsetPaths.append(subsetPath)
sequenceutils.writeFasta(sequences, subsetPath, subset)
i = i + 1
return subsetPaths
def initializeBackboneSequenceMapping(self):
if len(self.backboneTaxa) == 0:
backboneSubsetTaxonMap = {
i: subset
for i, subset in enumerate(self.subsets)
}
else:
backboneSubsetTaxonMap = {}
for taxon in self.backboneTaxa:
i = self.taxonSubsetMap[taxon]
backboneSubsetTaxonMap[i] = backboneSubsetTaxonMap.get(i, [])
backboneSubsetTaxonMap[i].append(taxon)
if Configs.constrain:
for i, subalignPath in enumerate(self.subalignmentPaths):
subalignment = sequenceutils.readFromFasta(subalignPath,
removeDashes=False)
for taxon in backboneSubsetTaxonMap.get(i, []):
self.backboneSubalignment[taxon] = subalignment[taxon]
else:
self.backboneSubalignment = self.unalignedSequences
def requestMafftBackbones(context):
numTaxa = max(1, int(Configs.mafftSize / len(context.subsetPaths)))
for n in range(Configs.mafftRuns):
unalignedFile = os.path.join(context.graph.workingDir,
"backbone_{}_unalign.txt".format(n + 1))
alignedFile = os.path.join(context.graph.workingDir,
"backbone_{}_mafft.txt".format(n + 1))
if os.path.exists(alignedFile):
Configs.log("Existing backbone file found: {}".format(alignedFile))
backbone = sequenceutils.readFromFasta(alignedFile)
context.backbonePaths.append(alignedFile)
else:
backbone = assignBackboneTaxa(context, numTaxa, unalignedFile)
backboneTask = external_tools.buildMafftAlignment(
unalignedFile, alignedFile)
context.backboneTasks.append(backboneTask)
if Configs.graphBuildHmmExtend:
context.backboneExtend.add(alignedFile)
else:
context.backboneTaxa.update(backbone)
task.submitTasks(context.backboneTasks)
def writeUnpackedAlignment(context):
graph = context.graph
filePath = context.outputFile
tempFile = os.path.join(os.path.dirname(filePath),
"temp_{}".format(os.path.basename(filePath)))
if os.path.exists(tempFile):
os.remove(tempFile)
clusterMap = {
path: [[] for c in graph.clusters]
for path in context.subalignmentPaths
}
for idx, cluster in enumerate(graph.clusters):
for b in cluster:
bsub, bpos = graph.matSubPosMap[b]
clusterMap[context.subalignmentPaths[bsub]][idx].append(bpos)
inserts = {path: [] for path in context.subalignmentPaths}
for b in graph.insertions:
bsub, bpos = graph.matSubPosMap[b]
inserts[context.subalignmentPaths[bsub]].append(bpos)
Configs.log("Assembling final alignment in {}".format(filePath))
inducedSubalignTasks = []
for bsub, subalignPath in enumerate(context.subalignmentPaths):
alignmentColumnsPath = os.path.join(
context.graph.workingDir,
"alignment_columns_{}".format(os.path.basename(subalignPath)))
with open(alignmentColumnsPath, 'w') as textFile:
textFile.write("{}\n".format(" ".join(
[str(c) for c in inserts[subalignPath]])))
for cluster in clusterMap[subalignPath]:
textFile.write("{}\n".format(" ".join(
[str(c) for c in cluster])))
inducedAlignPath = os.path.join(
graph.workingDir,
"induced_{}".format(os.path.basename(subalignPath)))
args = {
"alignmentColumnsPath": alignmentColumnsPath,
"subalignmentPath": subalignPath,
"outputFile": inducedAlignPath
}
inducedTask = task.Task(taskType="buildInducedSubalignment",
outputFile=args["outputFile"],
taskArgs=args)
inducedSubalignTasks.append(inducedTask)
#inducedTask.submitTask()
task.submitTasks(inducedSubalignTasks)
for inducedTask in task.asCompleted(inducedSubalignTasks):
inducedAlign = sequenceutils.readFromFasta(inducedTask.outputFile,
removeDashes=False)
Configs.log(
"Appending induced alignment, {} sequences of length {}..".format(
len(inducedAlign), len(next(iter(inducedAlign.values())).seq)))
sequenceutils.writeFasta(inducedAlign, tempFile, append=True)
os.remove(inducedTask.taskArgs["alignmentColumnsPath"])
os.remove(inducedTask.outputFile)
shutil.move(tempFile, filePath)
Configs.log("Wrote final alignment to {}".format(filePath))