def removeUnacceptableChromosomes(bedFilePaths: List[str], genomeFastaFilepath): acceptableChromosomes = getAcceptableChromosomes(genomeFastaFilepath) for bedFilePath in bedFilePaths: print("\nWorking in",os.path.basename(bedFilePath)) # Create an intermediate file to write acceptable entries to. acceptableChromBedFilePath = bedFilePath.rsplit('.', 1)[0] + "_acceptable_only.bed" removedEntries = 0 unacceptableChromosomes = list() # Read through the bed file, writing only entries with acceptable chromosomes to the output file. with open(bedFilePath, 'r') as bedFile: with open(acceptableChromBedFilePath, 'w') as acceptableChromBedFile: for line in bedFile: chromosome = line.split()[0] if chromosome in acceptableChromosomes: acceptableChromBedFile.write(line) else: removedEntries += 1 if chromosome not in unacceptableChromosomes: unacceptableChromosomes.append(chromosome) print("Unallowed chromosome found:",chromosome) # Rewrite the original file. print("Removed",removedEntries,"Entries. Rewriting original file...") os.replace(acceptableChromBedFilePath, bedFilePath)
def parseAlexandrov(bedInputFilePaths, genomeFilePath, nucPosFilePath): outputBedFilePaths = list() for bedInputFilePath in bedInputFilePaths: print("\nWorking in:", os.path.basename(bedInputFilePath)) # Get some important file system paths for the rest of the function and generate metadata. dataDirectory = os.path.dirname(bedInputFilePath) generateMetadata(os.path.basename(dataDirectory), getIsolatedParentDir(genomeFilePath), getIsolatedParentDir(nucPosFilePath), os.path.basename(bedInputFilePath), InputFormat.customBed, os.path.dirname(bedInputFilePath)) intermediateFilesDir = os.path.join(dataDirectory, "intermediate_files") checkDirs(intermediateFilesDir) # Get the list of acceptable chromosomes acceptableChromosomes = getAcceptableChromosomes(genomeFilePath) # Generate the output file. outputBedFilePath = generateFilePath( directory=intermediateFilesDir, dataGroup=getIsolatedParentDir(bedInputFilePath), dataType=DataTypeStr.customInput, fileExtension=".bed") # Write data to the output file. with open(bedInputFilePath, 'r') as bedInputFile: with open(outputBedFilePath, 'w') as outputBedFile: for line in bedInputFile: choppedUpLine = str(line).strip().split('\t') # Make sure we have a valid chromosome if ( "chr" + choppedUpLine[2] ) in acceptableChromosomes and not '/' in choppedUpLine[5]: # Convert the line to custom bed format. if choppedUpLine[5] == '-': choppedUpLine[5] = '*' if choppedUpLine[6] == '-': choppedUpLine[6] = '*' outputBedFile.write('\t'.join( ("chr" + choppedUpLine[2], str(int(choppedUpLine[3]) - 1), choppedUpLine[4], choppedUpLine[5], choppedUpLine[6], '.', choppedUpLine[0])) + '\n') # Add the output file to the list. outputBedFilePaths.append(outputBedFilePath) # Pass the data to the custome bed parser. print("\nPassing data to custom bed parser.\n") parseCustomBed(outputBedFilePaths, genomeFilePath, nucPosFilePath, False, False, False)
def parsePreparedInput(inputFilePaths: List[str], genomeFilePath, checkEachLine = True): for inputFilePath in inputFilePaths: print("\nWorking in",os.path.basename(inputFilePath)) # Perform some checks to make sure the input is formatted correctly. dataGroupName = getIsolatedParentDir(inputFilePath) inputFileBasename = os.path.basename(inputFilePath) inputFileContext = getContext(inputFilePath) if inputFileContext is None: raise UserInputError("No context is apparent from the given prepared input file.") if inputFileBasename.split('_'+inputFileContext)[0] != dataGroupName: raise InvalidPathError(inputFilePath, "Prepared input file is not named as expected given the data group name generated from the " "parent directory. Expected: \"" + dataGroupName + "\" immediately preceding the context definition " "but given file path is:") if not inputFileBasename.endswith(DataTypeStr.mutations + ".bed"): raise InvalidPathError(inputFilePath, "Prepared input file is not named properly to indicate the presence of mutation data. " "Expected a file ending in \"" + DataTypeStr.mutations + ".bed\" but given path is:") acceptableChromosomes = getAcceptableChromosomes(genomeFilePath) acceptableChromosomesFilePath = getAcceptableChromosomes(genomeFilePath, True) # Perform QA with the checkForErrors function print("Checking for errors in line formatting...") with open(inputFilePath, 'r') as inputFile: choppedUpLine = inputFile.readline().strip().split('\t') cohortDesignationPresent = len(choppedUpLine) == 7 checkForErrors(choppedUpLine, cohortDesignationPresent, acceptableChromosomes, acceptableChromosomesFilePath) if checkEachLine: for line in inputFile: choppedUpLine = line.strip().split('\t') checkForErrors(choppedUpLine, cohortDesignationPresent, acceptableChromosomes, acceptableChromosomesFilePath) # If everything else looks good, generate the metadata. This directory is now ready to go! print("Checks passed. Generating metadata, including mutation counts using a call to wc -l") generateMetadata(dataGroupName, getIsolatedParentDir(genomeFilePath), os.path.basename(inputFilePath), InputFormat.prepared, os.path.dirname(inputFilePath)) featureCounts = int(subprocess.check_output(("wc", "-l", inputFilePath), encoding = "UTF-8").split()[0]) Metadata(inputFilePath).addMetadata(Metadata.AddableKeys.mutCounts, featureCounts)
def __init__(self, inputDataFilePath: str, callParamsFilePath: str, genomeFilePath: str): self.inputDataFilePath = inputDataFilePath self.genomeFilePath = genomeFilePath # Get the list of acceptable chromosomes self.acceptableChromosomes = getAcceptableChromosomes(self.genomeFilePath) # Read in information from the callParams File. self.callParamsFilePath = callParamsFilePath self.expectedLocationsByLength = dict() self.acceptableBasesByLength = dict() with open(self.callParamsFilePath, 'r') as callParamsFile: for line in callParamsFile: choppedUpLine: List[str] = line.split() sequenceLength = int(choppedUpLine[0]) assert sequenceLength not in self.expectedLocationsByLength self.expectedLocationsByLength[sequenceLength] = list() expectedLocations = choppedUpLine[1].split(',') for expectedLocation in expectedLocations: if ':' not in expectedLocation: self.expectedLocationsByLength[sequenceLength].append((int(expectedLocation),int(expectedLocation))) else: expectedLocationStart, expectedLocationEnd = expectedLocation.split(':') if int(expectedLocationStart) > int(expectedLocationEnd): self.expectedLocationsByLength[sequenceLength].append((int(expectedLocationEnd),int(expectedLocationStart))) else: self.expectedLocationsByLength[sequenceLength].append((int(expectedLocationStart),int(expectedLocationEnd))) self.acceptableBasesByLength[sequenceLength] = choppedUpLine[2].split(',') # Initialize these values as empty until we know what form the input data is in. self.bigWigReadsFilePathPair = None self.bedGraphReadsFilePathPair = None self.bedInputFilePath = None # Initialize a variable to keep track of whether or not the input reads have been trimmed. self.readsHaveBeenTrimmed = False # Determine what form the input data is in and assign it accordingly. if self.inputDataFilePath.endswith(".bigWig"): self.bigWigReadsFilePathPair = getFilePathPair(self.inputDataFilePath) elif self.inputDataFilePath.endswith(".bedGraph"): self.bedGraphReadsFilePathPair = getFilePathPair(self.inputDataFilePath) elif self.inputDataFilePath.endswith(".bed"): self.bedInputFilePath = self.inputDataFilePath else: raise InvalidPathError(self.inputDataFilePath, "Input data file path is not in an acceptable format. " "Expected a bigWig, bedGraph, or bed file type but received:") self.setUpFileSystem()
def parseStandardBed(standardBedFilePaths: List[str], genomeFilePath): customBedOutputFilePaths = list( ) # The list of file paths to be passed to the custom bed parser. # Parse the given files into custom bed format. for standardBedFilePath in standardBedFilePaths: print("\nWorking in:", os.path.basename(standardBedFilePath)) if not os.path.basename(standardBedFilePath).endswith(".bed"): raise InvalidPathError( standardBedFilePath, "Given file does not appear to be in bed format. (missing \".bed\" extension)" ) # Store useful paths and names. localRootDirectory = os.path.dirname(standardBedFilePath) intermediateFilesDir = os.path.join(localRootDirectory, "intermediate_files") checkDirs(intermediateFilesDir) dataGroupName = getIsolatedParentDir(standardBedFilePath) # Generate the output file path and metadata customBedOutputFilePath = generateFilePath( directory=intermediateFilesDir, dataGroup=dataGroupName, dataType=DataTypeStr.customInput, fileExtension=".bed") customBedOutputFilePaths.append(customBedOutputFilePath) generateMetadata(dataGroupName, getIsolatedParentDir(genomeFilePath), os.path.basename(standardBedFilePath), InputFormat.standardBed, localRootDirectory) # Get the list of acceptable chromosomes. acceptableChromosomes = getAcceptableChromosomes(genomeFilePath) # Iterate through the standard bed file entries preparing them for custom-bed input. print("Converting entries for custom bed input...") with open(standardBedFilePath, 'r') as standardBedFile: with open(customBedOutputFilePath, 'w') as customBedOutputFile: for line in standardBedFile: choppedUpLine = line.strip().split("\t") # Make sure the lesion is in a valid chromosome. Otherwise, skip it. if not choppedUpLine[0] in acceptableChromosomes: continue choppedUpLine[3] = '.' choppedUpLine[4] = "OTHER" customBedOutputFile.write('\t'.join(choppedUpLine[:6]) + '\n') # Pass the generated files to the custom bed parser. parseCustomBed(customBedOutputFilePaths, genomeFilePath, False, False, False, False)
def __init__(self, ICGCFile: IO, genomeFilePath): self.ICGCFile = ICGCFile # The file containing the ICGC data self.acceptableChromosomes = getAcceptableChromosomes(genomeFilePath) self.finishedDonors = list( ) # A list of donors to make sure we don't encounter one more than once. self.currentDonor = '' # The donorID currently being read for mutation data. self.currentDonorMutations = dict( ) # A dictionary of mutations unique to the current donor, to avoid writing duplicate mutations. self.previousDonorMutations: List[ICGCMutation] = list( ) # A list that keeps the last donor's mutations in order to write data on individual donors all at once.
def countInBindingMotifs(mutationFilePaths, bindingMotifsFilePaths): bindingMotifsMutationCountsFilePaths = list( ) # A list of paths to the output files generated by the function # Loop through each given mutation file path, creating a corresponding binding motifs mutation count file for each. for mutationFilePath in mutationFilePaths: for bindingMotifsFilePath in bindingMotifsFilePaths: print("\nWorking with", os.path.basename(mutationFilePath), "and", os.path.basename(bindingMotifsFilePath)) # Make sure we have the expected file type. if not DataTypeStr.mutations in os.path.basename(mutationFilePath): raise ValueError("Mutation file should have \"" + DataTypeStr.mutations + "\" in the name.") # Get metadata and use it to generate a path to the nucleosome positions file. metadata = Metadata(mutationFilePath) # Generate the output file path for mutation counts. binder = os.path.basename(bindingMotifsFilePath).rsplit( "binding_motifs", 1)[0] if "binding_motifs" not in os.path.basename(bindingMotifsFilePath): warnings.warn( "\"binding_motifs\" not found in basename of binding motifs file. The output file's name is probably a garbled mess." ) bindingMotifsMutationCountsFilePath = generateFilePath( directory=metadata.directory, dataGroup=metadata.dataGroupName, fileExtension=".tsv", dataType=binder + "binding_motif_mutation_counts") bindingMotifsMutationCountsFilePaths.append( bindingMotifsMutationCountsFilePath) # Ready, set, go! counter = CountsFileGenerator( mutationFilePath, bindingMotifsFilePath, bindingMotifsMutationCountsFilePath, getAcceptableChromosomes(metadata.genomeFilePath)) counter.count() counter.writeResults() return bindingMotifsMutationCountsFilePaths
def generateMutationBackground(mutationFilePaths, backgroundContextNum): mutationBackgroundFilePaths = list( ) # A list of paths to the output files generated by the function # A dictionary for converting context numbers to text. contextNumToText = { 1: "singlenuc", 2: "dinuc", 3: "trinuc", 4: "quadrunuc", 5: "pentanuc", 6: "hexanuc" } for mutationFilePath in mutationFilePaths: # Retrieve metadata metadata = Metadata(mutationFilePath) intermediateFilesDirectory = os.path.join(metadata.directory, "intermediate_files") # If necessary, adjust the context for files with even-length features. if getContext(mutationFilePath, asInt=True) % 2 == 0: thisBackgroundContextNum = backgroundContextNum + 1 else: thisBackgroundContextNum = backgroundContextNum # Set the name of the type of context being used. assert thisBackgroundContextNum in contextNumToText, "Unexpected background context number: " + str( thisBackgroundContextNum) contextText = contextNumToText[thisBackgroundContextNum] # Get the list of acceptable chromosomes acceptableChromosomes = getAcceptableChromosomes( metadata.genomeFilePath) print("\nWorking in:", os.path.split(mutationFilePath)[1]) if not DataTypeStr.mutations in os.path.split(mutationFilePath)[1]: raise InvalidPathError( mutationFilePath, "Given mutation file does not have \"" + DataTypeStr.mutations + "\" in the name.", postPathMessage= "Are you sure you inputted a file from the mutperiod pipeline?" ) # Generate the file path for the genome context frequency file. genomeContextFrequencyFilePath = generateFilePath( directory=os.path.dirname(metadata.genomeFilePath), dataGroup=metadata.genomeName, context=contextText, dataType="frequency", fileExtension=".tsv") # Generate the file path for the mutation context frequency file. mutationContextFrequencyFilePath = generateFilePath( directory=intermediateFilesDirectory, dataGroup=metadata.dataGroupName, context=contextText, dataType="mutation_frequencies", fileExtension=".tsv") # Generate the file path for the background mutation rate file. mutationBackgroundFilePath = generateFilePath( directory=metadata.directory, dataGroup=metadata.dataGroupName, context=contextText, dataType=DataTypeStr.mutBackground, fileExtension=".tsv") # If the genome context frequency file doesn't exist, create it. if not os.path.exists(genomeContextFrequencyFilePath): print("Genome", contextText, "context frequency file not found at path:", genomeContextFrequencyFilePath) print("Generating genome " + contextText + " context frequency file...") generateGenomeContextFrequencyFile(metadata.genomeFilePath, genomeContextFrequencyFilePath, thisBackgroundContextNum, contextText, acceptableChromosomes) # Create a directory for intermediate files if it does not already exist... if not os.path.exists(intermediateFilesDirectory): os.mkdir(intermediateFilesDirectory) # Create the mutation context frequency file. print("Generating mutation context frequency file...") generateMutationContextFrequencyFile(mutationFilePath, mutationContextFrequencyFilePath, thisBackgroundContextNum, contextText, acceptableChromosomes) # Generate the mutation background file. generateMutationBackgroundFile(genomeContextFrequencyFilePath, mutationContextFrequencyFilePath, mutationBackgroundFilePath, contextText) mutationBackgroundFilePaths.append(mutationBackgroundFilePath) return mutationBackgroundFilePaths
def parseDeaminationData(cPDFilePaths: List[str], deaminationFilePaths: List[str], genomeFastaFilePath): """ See script header. """ acceptableChromosomes = getAcceptableChromosomes(genomeFastaFilePath) for cPDFilePath in cPDFilePaths: print("\nWorking in", os.path.basename(cPDFilePath)) # Create a name for the parsed output file. cPDParsedFilePath = cPDFilePath.rsplit('.', 1)[0] + "_parsed.bed" # Create a path to the output file with only cytosine positions. cPDCytosinePositionsFilePath = cPDFilePath.rsplit( '.', 1)[0] + "_cytosines.bed" with open(cPDFilePath, 'r') as cPDFile: with open(cPDParsedFilePath, 'w') as cPDParsedFile: print("Parsing original file...") cPDFile.readline() # Skip the header line. for line in cPDFile: choppedUpLine = line.split() # Record position information, inferring strand from the given gap sequencing value # and extending positions to encompass the full CPD sequence. chromosome = choppedUpLine[0] # Check for invalid chromosomes if chromosome not in acceptableChromosomes: print("Skipping invalid chromosome:", chromosome) continue if isPurine(choppedUpLine[3]): strand = '-' position0 = str(int(choppedUpLine[1]) - 1) position1 = choppedUpLine[2] else: strand = '+' position0 = choppedUpLine[1] position1 = str(int(choppedUpLine[2]) + 1) # Make sure there is aggreement among the weird sequence columns as to the CPD sequence assert choppedUpLine[5][-2:] == choppedUpLine[6][ -2:] == choppedUpLine[7][:2] == choppedUpLine[ 8][:2], line cPD = choppedUpLine[5][-2:] cPDParsedFile.write('\t'.join( (chromosome, position0, position1, cPD, '.', strand)) + '\n') # Derive the sequences directly from the positions and make sure it matches the cPD value obtained previously. # At the same time, create the file with only the cytosine positions in CPDs. print("Deriving sequence context from given fasta file...") addSequenceToBed(cPDParsedFilePath, genomeFastaFilePath, 4) with open(cPDParsedFilePath, 'r') as cPDParsedFile: with open(cPDCytosinePositionsFilePath, 'w') as cPDCytosinePositionsFile: print( "Validating original file and trimming to single base cytosine positions..." ) for line in cPDParsedFile: choppedUpLine = line.split() assert choppedUpLine[3] == choppedUpLine[4], line for i, base in enumerate(choppedUpLine[3]): if base == 'C': if choppedUpLine[5] == '+': position0 = str(int(choppedUpLine[1]) + i) else: position0 = str(int(choppedUpLine[1]) + 1 - i) position1 = str(int(position0) + 1) cPDCytosinePositionsFile.write('\t'.join( (choppedUpLine[0], position0, position1, '.', '.', choppedUpLine[5])) + '\n') for deaminationFilePath in deaminationFilePaths: print("\nWorking in", os.path.basename(deaminationFilePath)) # Create a name for the parsed output file. deaminationParsedFilePath = deaminationFilePath.rsplit( '.', 1)[0] + "_parsed.bed" # Create a path to the output file with only cytosine positions in dipy contexts. dipyDeaminationPositionsFilePath = deaminationFilePath.rsplit( '.', 1)[0] + "_dipy_cytosines.bed" with open(deaminationFilePath, 'r') as deaminationFile: with open(deaminationParsedFilePath, 'w') as deaminationParsedFile: print("Parsing original file...") deaminationFile.readline() # Skip the header line. for line in deaminationFile: choppedUpLine = line.split() # Check for non-cytosine positions, and ignore them if found. if choppedUpLine[3] in ('A', 'T'): continue # Record position information for all other rows. Expand to trinucleotide context. chromosome = choppedUpLine[0] # Check for invalid chromosomes if chromosome not in acceptableChromosomes: print("Skipping invalid chromosome:", chromosome) continue position0 = str(int(choppedUpLine[1]) - 1) position1 = str(int(choppedUpLine[2]) + 1) if choppedUpLine[3] == 'C': strand = '+' else: strand = '-' trinuc = choppedUpLine[5] deaminationParsedFile.write('\t'.join( (chromosome, position0, position1, trinuc, '.', strand)) + '\n') # Derive the trinucleotide context sequences directly from the positions and make sure it matches the sequence obtained from the file. # At the same time, create the file with only the cytosine positions with an adjacent pyrimidine. print("Deriving sequence context from given fasta file...") addSequenceToBed(deaminationParsedFilePath, genomeFastaFilePath, substitutionPosition=4) with open(deaminationParsedFilePath, 'r') as deaminationParsedFile: with open(dipyDeaminationPositionsFilePath, 'w') as dipyDeaminationPositionsFile: print( "Validating original file and trimming cytosines without adjacent pyrimidines..." ) for line in deaminationParsedFile: choppedUpLine = line.split() assert choppedUpLine[3] == choppedUpLine[4], line if isPurine(choppedUpLine[3][0]) and isPurine( choppedUpLine[3][2]): continue else: position0 = str(int(choppedUpLine[1]) + 1) position1 = str(int(choppedUpLine[2]) - 1) dipyDeaminationPositionsFile.write('\t'.join( (choppedUpLine[0], position0, position1, '.', '.', choppedUpLine[5])) + '\n')
def countNucleosomePositionMutations(mutationFilePaths, nucleosomeMapNames, countSingleNuc, countNucGroup, linkerOffset): # Check for the special case where a nucleosome map is being counted against itself to determine the nucleosome repeat length. if (len(mutationFilePaths) == 1 and len(nucleosomeMapNames) == 1 and os.path.basename(mutationFilePaths[0]).rsplit( '.', 1)[0] == nucleosomeMapNames[0]): nucleosomeMapFilePath = mutationFilePaths[0] nucleosomeMapName = nucleosomeMapNames[0] print("Counting nucleosome map", nucleosomeMapName, "against itself in a 1000 bp radius.") countsFilePath = generateFilePath( directory=os.path.dirname(nucleosomeMapFilePath), dataGroup=nucleosomeMapName, usesNucGroup=True, fileExtension=".tsv", dataType="self_" + DataTypeStr.rawNucCounts) acceptableChromosomes = getAcceptableChromosomes( os.path.dirname(os.path.dirname(nucleosomeMapFilePath))) counter = NucleosomesInNucleosomesCounter( nucleosomeMapFilePath, nucleosomeMapFilePath, countsFilePath, encompassingFeatureExtraRadius=1000, acceptableChromosomes=acceptableChromosomes) counter.count() return [countsFilePath] if not (countSingleNuc or countNucGroup): raise UserInputError( "Must count in either a single nucleosome or group nucleosome radius." ) nucleosomeMutationCountsFilePaths = list( ) # A list of paths to the output files generated by the function nucleosomeMapSortingChecked = False # Use this to make sure files are checked for sorting only once. # Loop through each given mutation file path, creating a corresponding nucleosome mutation count file for each. for mutationFilePath in mutationFilePaths: print("\nWorking with", os.path.split(mutationFilePath)[1]) # Make sure we have the expected file type. if not DataTypeStr.mutations in os.path.basename(mutationFilePath): raise InvalidPathError( mutationFilePath, "Given mutation file does not have \"" + DataTypeStr.mutations + "\" in the name.", postPathMessage= "Are you sure you inputted a file from the mutperiod pipeline?" ) for nucleosomeMapName in nucleosomeMapNames: print("Counting with nucleosome map:", nucleosomeMapName) # Generate the path to the nucleosome-map-specific directory. nucleosomeMapDataDirectory = os.path.join( os.path.dirname(mutationFilePath), nucleosomeMapName) checkDirs(nucleosomeMapDataDirectory) # Check to see if the metadata for this directory has been generated before, and if not, set it up! if not os.path.exists( os.path.join(nucleosomeMapDataDirectory, ".metadata")): print("No metadata found. Generating...") parentMetadata = Metadata(mutationFilePath) # Check to see if the data name should be altered by this nucleosome map. dataGroupName = parentMetadata.dataGroupName dataGroupNameSuffixFilePath = os.path.join( os.path.dirname(parentMetadata.genomeFilePath), nucleosomeMapName, "append_to_data_name.txt") if os.path.exists(dataGroupNameSuffixFilePath): with open(dataGroupNameSuffixFilePath ) as dataGroupNameSuffixFile: dataGroupName += dataGroupNameSuffixFile.readline( ).strip() generateMetadata( dataGroupName, parentMetadata.genomeName, os.path.join("..", parentMetadata.localParentDataPath), parentMetadata.inputFormat, nucleosomeMapDataDirectory, *parentMetadata.cohorts, callParamsFilePath=parentMetadata.callParamsFilePath, associatedNucleosomePositions=nucleosomeMapName) # Get metadata and use it to generate a path to the nucleosome positions file. metadata = Metadata(nucleosomeMapDataDirectory) # Get the list of acceptable chromosomes acceptableChromosomes = getAcceptableChromosomes( metadata.genomeFilePath) # Generate the counts file for a single nucleosome region if requested. if countSingleNuc: # Generate the output file path nucleosomeMutationCountsFilePath = generateFilePath( directory=metadata.directory, dataGroup=metadata.dataGroupName, linkerOffset=linkerOffset, fileExtension=".tsv", dataType=DataTypeStr.rawNucCounts) # Ready, set, go! print( "Counting mutations at each nucleosome position in a 73 bp radius +", str(linkerOffset), "bp linker DNA.") counter = MutationsInNucleosomesCounter( mutationFilePath, metadata.baseNucPosFilePath, nucleosomeMutationCountsFilePath, encompassingFeatureExtraRadius=73 + linkerOffset, acceptableChromosomes=acceptableChromosomes, checkForSortedFiles=(True, not nucleosomeMapSortingChecked)) counter.count() nucleosomeMutationCountsFilePaths.append( nucleosomeMutationCountsFilePath) # Generate the counts file for a nucleosome group region if requested. if countNucGroup: # Generate the output file path nucleosomeMutationCountsFilePath = generateFilePath( directory=metadata.directory, dataGroup=metadata.dataGroupName, usesNucGroup=True, fileExtension=".tsv", dataType=DataTypeStr.rawNucCounts) # Ready, set, go! print( "Counting mutations at each nucleosome position in a 1000 bp radius." ) counter = MutationsInNucleosomesCounter( mutationFilePath, metadata.baseNucPosFilePath, nucleosomeMutationCountsFilePath, encompassingFeatureExtraRadius=1000, acceptableChromosomes=acceptableChromosomes, checkForSortedFiles=(True, not nucleosomeMapSortingChecked)) counter.count() nucleosomeMutationCountsFilePaths.append( nucleosomeMutationCountsFilePath) nucleosomeMapSortingChecked = True return nucleosomeMutationCountsFilePaths
def autoAcquireAndQACheck(bedInputFilePath: str, genomeFilePath, autoAcquiredFilePath, onlySingleBaseSubs, includeIndels): print( "Checking custom bed file for formatting and auto-acquire requests...") # To start, assume that no sequences need to be acquired, and do it on the fly if need be. autoAcquiring = False autoAcquireFastaIterator = None fastaEntry = None cohortDesignationPresent = None # Unless indels are included, determine the context of the feqtures in the file. if includeIndels: context = 0 else: context = None # Get the list of acceptable chromosomes acceptableChromosomes = getAcceptableChromosomes(genomeFilePath) acceptableChromosomesFilePath = getAcceptableChromosomes( genomeFilePath, True) # Create a temporary file to write the data to (potentially after auto-acquiring). # Will replace original file at the end if auto-acquiring occurred. temporaryBedFilePath = bedInputFilePath + ".tmp" # Iterate through the input file one line at a time, checking the format of each entry and looking for auto-acquire requests. with open(bedInputFilePath, 'r') as bedInputFile: with open(temporaryBedFilePath, 'w') as temporaryBedFile: for line in bedInputFile: choppedUpLine = str(line).strip().split('\t') # If it isn't already, initialize the cohortDesignationPresent variable. if cohortDesignationPresent is None: cohortDesignationPresent = len(choppedUpLine) == 7 # Check for possible error states. checkForErrors(choppedUpLine, cohortDesignationPresent, acceptableChromosomes, acceptableChromosomesFilePath) # If this is the first entry requiring auto-acquiring, generate the required fasta file. if (not autoAcquiring and (choppedUpLine[3] == '.' or choppedUpLine[4] == '.' or (choppedUpLine[5] == '.' and choppedUpLine[3] != '*'))): print( "Found line with auto-acquire requested. Generating fasta..." ) autoAcquiring = True bedToFasta(bedInputFilePath, genomeFilePath, autoAcquiredFilePath) autoAcquiredFile = open(autoAcquiredFilePath, 'r') autoAcquireFastaIterator = FastaFileIterator( autoAcquiredFile) fastaEntry = autoAcquireFastaIterator.readEntry() print("Continuing...") # Check for any base identities that need to be auto-acquired. if choppedUpLine[3] == '.': # Find the equivalent fasta entry. while not equivalentEntries(fastaEntry, choppedUpLine): assert not autoAcquireFastaIterator.eof, ( "Reached end of fasta file without finding a match for: ", ' '.join(choppedUpLine)) fastaEntry = autoAcquireFastaIterator.readEntry() # Set the sequence. choppedUpLine[3] = fastaEntry.sequence # Check for any strand designations that need to be auto-acquired. # Also, make sure this isn't an insertion, in which case the strand designation cannot be determined. if choppedUpLine[5] == '.' and choppedUpLine[3] != '*': # Find the equivalent fasta entry. while not equivalentEntries(fastaEntry, choppedUpLine): assert not autoAcquireFastaIterator.eof, ( "Reached end of fasta file without finding a match for: ", ' '.join(choppedUpLine)) fastaEntry = autoAcquireFastaIterator.readEntry() # Determine which strand is represented. if fastaEntry.sequence == choppedUpLine[3]: choppedUpLine[5] = '+' elif fastaEntry.sequence == reverseCompliment( choppedUpLine[3]): choppedUpLine[5] = '-' else: assert False, ( "The given sequence " + choppedUpLine[3] + " for location " + fastaEntry.sequenceName + ' ' + "does not match the corresponding sequence in the given genome, or its reverse compliment." ) # Change any '.' characters in the "altered to" column to "OTHER" if choppedUpLine[4] == '.': choppedUpLine[4] = "OTHER" # Determine the sequence context of the line and whether or not it matches the sequence context for other. # Skip this if the file is "mixed", this line is an indel, or only single base substitutions are allowed and this line isn't one. if (not context == 0 and not (choppedUpLine[3] == '*' or choppedUpLine[4] == '*') and (not onlySingleBaseSubs or isSingleBaseSubstitution(choppedUpLine))): thisContext = len(choppedUpLine[3]) if context is None: context = thisContext elif thisContext != context: context = 0 # Write the current line to the temporary bed file. temporaryBedFile.write('\t'.join(choppedUpLine) + '\n') # If any lines were auto-acquired, replace the input bed file with the temporary bed file. (Which has auto-acquires) if autoAcquiring: print( "Overwriting custom bed input with auto-acquired bases/strand designations." ) os.replace(temporaryBedFilePath, bedInputFilePath) # Otherwise, just delete the temporary file. else: os.remove(temporaryBedFilePath) if context > 6: context = float("inf") return context
def parseKucabCompendium(kucabSubstitutionsFilePaths: List[str], genomeFilePath, nucPosFilePath, includeAllPAHs): for kucabSubstitutionsFilePath in kucabSubstitutionsFilePaths: print("\nWorking in:", os.path.basename(kucabSubstitutionsFilePath)) if not kucabSubstitutionsFilePath.endswith("final.txt"): raise InvalidPathError( kucabSubstitutionsFilePath, "Given kucab input file does not end in \"final.txt\":") # Prepare the output file path. localRootDirectory = os.path.dirname(kucabSubstitutionsFilePath) dataGroupName = getIsolatedParentDir(kucabSubstitutionsFilePath) if includeAllPAHs: outputDirectory = os.path.join(localRootDirectory, "all_PAHs") dataGroupName += "_all_PAHs" else: dataGroupName += "_smoker_lung" outputDirectory = os.path.join(localRootDirectory, "smoker_lung") # Make sure the data directory exists. if not os.path.exists(outputDirectory): os.mkdir(outputDirectory) # Generate the output file path and metadata outputTrinucBedFilePath = generateFilePath( directory=outputDirectory, dataGroup=dataGroupName, context="trinuc", dataType=DataTypeStr.mutations, fileExtension=".bed") generateMetadata( dataGroupName, getIsolatedParentDir(genomeFilePath), getIsolatedParentDir(nucPosFilePath), os.path.join("..", os.path.basename(kucabSubstitutionsFilePath)), outputDirectory) # Get the list of acceptable chromosomes acceptableChromosomes = getAcceptableChromosomes(genomeFilePath) # These are the designations for PAH mutation signatures, the ones related to tobacco smoke that we want to study. PAHDesignations = ("MSM0.54", "MSM0.26", "MSM0.92", "MSM0.2", "MSM0.42", "MSM0.74", "MSM0.103" "MSM0.14", "MSM0.82", "MSM0.130", "MSM0.12", "MSM0.132", "MSM0.13", "MSM0.96") # These designations specifically mimic the indel signature in smokers' lung cancer tumors. LungCancerSpecificDesignations = ("MSM0.26", "MSM0.92", "MSM0.2", "MSM0.103", "MSM0.14") # Set the designations that will be used to collect data based on the input to the function. if includeAllPAHs: relevantDesignations = PAHDesignations else: relevantDesignations = LungCancerSpecificDesignations print("Reading data and writing to trinuc bed file...") with open(kucabSubstitutionsFilePath, 'r') as kucabSubstitutionsFile: with open(outputTrinucBedFilePath, 'w') as outputTrinucBedFile: firstLineFlag = True for line in kucabSubstitutionsFile: # Skip the first line with headers. if firstLineFlag: firstLineFlag = False continue # The lines are separated by tabs. The relevant data have the following indices in a tab-separated list: # 15: mutagen designation # 4: Chromosome # 5: Start Pos (1 base) # 6: Reference base # 7: Mutated base # 13: pre-base context # 14: post-base context choppedUpLine = line.strip().split('\t') # Skip the mutation if it does not belong to the relevant group. if not choppedUpLine[15] in relevantDesignations: continue # Compile the necessary information for the bed file. chromosome = "chr" + choppedUpLine[4] # Handle the weird chromsome formatting and then check for invalid chromosomes. if chromosome == "chr23": chromosome = "chrX" if chromosome == "chr24": chromosome = "chrY" if not chromosome in acceptableChromosomes: continue startPos1Base = choppedUpLine[5] startPos0Base = str(int(startPos1Base) - 1) mutatedFrom = choppedUpLine[6] mutatedTo = choppedUpLine[7] trinucContext = ''.join( (choppedUpLine[13], mutatedFrom, choppedUpLine[14])) # If the mutated base is listed as arising from a purine, flip the mutation and the strand. if isPurine(mutatedFrom): mutation = reverseCompliment( mutatedFrom) + '>' + reverseCompliment(mutatedTo) strand = '-' trinucContext = reverseCompliment(trinucContext) else: mutation = mutatedFrom + '>' + mutatedTo strand = '+' # Write the information to the trinuc bed file. outputTrinucBedFile.write('\t'.join( (chromosome, startPos0Base, startPos1Base, trinucContext, mutation, strand)) + '\n') # Sort the output file. print("Sorting output file...") subprocess.run(("sort", "-k1,1", "-k2,2n", outputTrinucBedFilePath, "-o", outputTrinucBedFilePath), check=True)