def main(): #python radiaCompare.py TCGA-AB-2995 12 ../data/test/TCGA-AB-2995.vcf ../data/test/TCGA-AB-2995.vcf -c "SOM=Somatic" -s ../stats/radia/cmpRadBB.tab --log=DEBUG # create the usage statement usage = "usage: python %prog id chrom radFile compareFile [Options]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option("-c", "--compareList", dest="compareList", metavar="COMPARE_LIST", help="a comma separated list of key/values comparisons where the key is in RADIA and the value is in the compare file") i_cmdLineParser.add_option("-s", "--statsFilename", dest="statsFilename", metavar="STATS_FILE", help="the name of the stats file, sys.stdout by default") i_cmdLineParser.add_option("-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default") i_cmdLineParser.add_option("-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDERR by default") i_cmdLineParser.add_option("-o", "--overlapFilename", dest="overlapFilename", metavar="OVERLAP_FILE", help="the name of the overlap file") i_cmdLineParser.add_option("-n", "--nonOverlapFilename", dest="nonOverlapFilename", metavar="NON_OVERLAP_FILE", help="the name of the non-overlap file") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(4,17,1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = str(i_cmdLineArgs[0]) i_chr = str(i_cmdLineArgs[1]) i_radiaFilename = str(i_cmdLineArgs[2]) i_compareFilename = str(i_cmdLineArgs[3]) # get the optional params with default values i_logLevel = i_cmdLineOptions.logLevel # try to get any optional parameters with no defaults # check for any errors writeFilenameList = [] readFilenameList = [i_radiaFilename, i_compareFilename] i_statsFilename = None i_logFilename = None i_compareString = None i_overlapFilename = None i_nonOverlapFilename = None i_compareDict = collections.defaultdict(list) if (i_cmdLineOptions.overlapFilename != None): i_overlapFilename = str(i_cmdLineOptions.overlapFilename) writeFilenameList += [i_overlapFilename] if (i_cmdLineOptions.nonOverlapFilename != None): i_nonOverlapFilename = str(i_cmdLineOptions.nonOverlapFilename) writeFilenameList += [i_nonOverlapFilename] if (i_cmdLineOptions.statsFilename != None): i_statsFilename = str(i_cmdLineOptions.statsFilename) writeFilenameList += [i_statsFilename] if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) writeFilenameList += [i_logFilename] if (i_cmdLineOptions.compareList != None): i_compareString = str(i_cmdLineOptions.compareList) i_compareList = i_compareString.split(",") for keyValue in i_compareList: (key, value) = keyValue.split("=") i_compareDict[key] = value # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # do some debugging if (i_debug): logging.debug("id=%s", i_id) logging.debug("chr=%s", i_chr) logging.debug("radiaFile=%s", i_radiaFilename) logging.debug("overlapFilename=%s" % i_overlapFilename) logging.debug("nonOverlapFilename=%s" % i_nonOverlapFilename) logging.debug("compareFile=%s", i_compareFilename) logging.debug("statsFile=%s", i_statsFilename) logging.debug("logLevel=%s", i_logLevel) logging.debug("logFile=%s", i_logFilename) logging.debug("compareDict=%s", i_compareDict) if (not radiaUtil.check_for_argv_errors(None, readFilenameList, writeFilenameList)): sys.exit(1) compare_events(i_id, i_chr, i_radiaFilename, i_compareFilename, i_statsFilename, i_overlapFilename, i_nonOverlapFilename, i_compareDict, i_debug) return
def main(): # command for running this on a small test case: #python filterByBlat.py TCGA-00-4454 7 ../data/test/TCGA-00-4454_EGFR.vcf ../data/test/TCGA-00-4454_EGFR.fa ../data/test/TCGA-00-4454_EGFR.psl --dnaNormalFilename=../data/test/TCGA-00-4454_EGFR.reads startTime = time.time() # create the usage statement usage = "usage: python %prog id chrom vcfFile blatInputFile blatOutputFile [Options]" i_cmdLineParser = OptionParser(usage=usage) # add the optional parameters i_cmdLineParser.add_option("-c", "--allVCFCalls", action="store_false", default=True, dest="passedVCFCallsOnly", help="by default only the VCF calls that have passed all filters thus far are processed, include this argument if all of the VCF calls should be processed") i_cmdLineParser.add_option("-k", "--keepPreviousFilters", action="store_true", default=False, dest="keepPreviousFilters", help="by default the previous filters are overwritten with the blat filter, include this argument if the previous filters should be kept") i_cmdLineParser.add_option("-o", "--outputFilename", dest="outputFilename", metavar="OUTPUT_FILE", help="the name of the output file, STDOUT by default") i_cmdLineParser.add_option("-b", "--blatOutputFormat", dest="blatOutputFormat", metavar="OUTPUT_FORMAT", default="BLAST", help="the BLAT output format, BLAST by default") i_cmdLineParser.add_option("-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default") i_cmdLineParser.add_option("-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") i_cmdLineParser.add_option("-n", "--blatDnaNormalReads", action="store_true", default=False, dest="blatDnaNormalReads", help="include this argument if the normal DNA reads should be processed") i_cmdLineParser.add_option("-x", "--blatRnaNormalReads", action="store_true", default=False, dest="blatRnaNormalReads", help="include this argument if the normal RNA reads should be processed") i_cmdLineParser.add_option("-t", "--blatDnaTumorReads", action="store_true", default=False, dest="blatDnaTumorReads", help="include this argument if the tumor DNA reads should be processed") i_cmdLineParser.add_option("-r", "--blatRnaTumorReads", action="store_true", default=False, dest="blatRnaTumorReads", help="include this argument if the tumor RNA reads should be processed") i_cmdLineParser.add_option("-d", "--readDepthCutoff", type="int", default=int(4), dest="readDepthCutoff", metavar="READ_DP_CUTOFF", help="the minimum number of valid reads that are necessary, %default by default") i_cmdLineParser.add_option("-p", "--readPercentCutoff", type="float", default=float(0.10), dest="readPercentCutoff", metavar="READ_PERCENT_CUTOFF", help="the minimum percentage of valid reads that are necessary, %default by default") #i_cmdLineParser.add_option("-e", "--eValueCutoff", type="float", default=float(10e-6), dest="eValueCutoff", metavar="EVAL_CUTOFF", help="the e-value cutoff for determining if a blat hit is significant, %default by default") #i_cmdLineParser.add_option("-u", "--upperIdentityCutoff", type="float", default=float(0.95), dest="upperIdentityCutoff", metavar="UPPER_CUTOFF", help="the upper cutoff for the match length adjusted identity to determine if a blat hit is significant, %default by default") #i_cmdLineParser.add_option("-l", "--lowerIdentityCutoff", type="float", default=float(0.5), dest="lowerIdentityCutoff", metavar="LOWER_CUTOFF", help="the lower cutoff for the match length adjusted identity to determine if a second blat hit is significant, %default by default") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(5,27,1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = str(i_cmdLineArgs[0]) i_vcfFilename = str(i_cmdLineArgs[1]) i_blatInputFilename = str(i_cmdLineArgs[2]) i_blatOutputFilename = str(i_cmdLineArgs[3]) # get the optional params with default values i_passedVCFCallsOnlyFlag = i_cmdLineOptions.passedVCFCallsOnly i_keepPreviousFiltersFlag = i_cmdLineOptions.keepPreviousFilters i_blatOutputFormat = i_cmdLineOptions.blatOutputFormat i_logLevel = i_cmdLineOptions.logLevel i_readDepthCutoff = i_cmdLineOptions.readDepthCutoff i_readPercentCutoff = i_cmdLineOptions.readPercentCutoff #i_eValueCutoff = i_cmdLineOptions.eValueCutoff #i_upperIdentityCutoff = i_cmdLineOptions.upperIdentityCutoff #i_lowerIdentityCutoff = i_cmdLineOptions.lowerIdentityCutoff i_blatDnaNormalReads = i_cmdLineOptions.blatDnaNormalReads i_blatDnaTumorReads = i_cmdLineOptions.blatDnaTumorReads i_blatRnaNormalReads = i_cmdLineOptions.blatRnaNormalReads i_blatRnaTumorReads = i_cmdLineOptions.blatRnaTumorReads # try to get any optional parameters with no defaults i_outputFilename = None i_logFilename = None if (i_cmdLineOptions.outputFilename != None): i_outputFilename = str(i_cmdLineOptions.outputFilename) if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel < logging.WARNING) # output some debug info if (i_debug): logging.debug("id=%s", i_id) logging.debug("vcfFilename=%s", i_vcfFilename) logging.debug("blatInputFilename=%s", i_blatInputFilename) logging.debug("blatOutputFilename=%s", i_blatOutputFilename) logging.debug("passedCallsOnly? %s", i_passedVCFCallsOnlyFlag) logging.debug("keepPreviousFiltersFlag? %s", i_keepPreviousFiltersFlag) logging.debug("blatOutputFormat=%s", i_blatOutputFormat) logging.debug("blatDnaNormal? %s", i_blatDnaNormalReads) logging.debug("blatDnaTumor? %s", i_blatDnaTumorReads) logging.debug("blatRnaNormal? %s", i_blatRnaNormalReads) logging.debug("blatRnaTumor? %s", i_blatRnaTumorReads) logging.debug("readDepthCutoff=%s", i_readDepthCutoff) logging.debug("readPerentCutoff=%s", i_readPercentCutoff) # check for any errors i_writeFilenameList = [] if (i_outputFilename != None): i_writeFilenameList = [i_outputFilename] if (i_logFilename != None): i_writeFilenameList = [i_logFilename] i_readFilenameList = [i_vcfFilename, i_blatInputFilename, i_blatOutputFilename] if (not radiaUtil.check_for_argv_errors(None, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # open the output stream i_outputFileHandler = None if (i_outputFilename != None): i_outputFileHandler = get_write_fileHandler(i_outputFilename) # get the BLAT results i_blatCoordinateDict = parse_blat_output(i_blatOutputFilename, i_blatOutputFormat, i_debug) # get the VCF generator i_vcfGenerator = get_vcf_data(i_vcfFilename, i_passedVCFCallsOnlyFlag, i_debug) for (vcfChr, vcfStopCoordinate, vcfId, vcfRef, vcfAlt, vcfScore, vcfFilterSet, vcfInfoDict, restOfLine) in i_vcfGenerator: if (i_debug): logging.debug("VCF Data: %s %s %s %s %s %s %s %s %s", vcfChr, str(vcfStopCoordinate), vcfId, vcfRef, vcfAlt, vcfScore, str(vcfFilterSet), str(vcfInfoDict), restOfLine) modTypes = vcfInfoDict["MT"] modTypeFilters = dict() atLeastOnePass = False for modType in modTypes: blatHitsDict = dict() blatOverallReadDepth = 0 numValidReads = 0 if (modType == "NOR_EDIT" and i_blatRnaNormalReads): if ("rnaNormal" in i_blatCoordinateDict[vcfChr + "_" + str(vcfStopCoordinate)]): # for each coordinate, get a dict of reads and corresponding blat hits blatHitsDict = i_blatCoordinateDict[vcfChr + "_" + str(vcfStopCoordinate)]["rnaNormal"] elif ((modType == "SOM" or modType == "TUM_EDIT") and i_blatRnaTumorReads): if ("rnaTumor" in i_blatCoordinateDict[vcfChr + "_" + str(vcfStopCoordinate)]): # for each coordinate, get a dict of reads and corresponding blat hits blatHitsDict = i_blatCoordinateDict[vcfChr + "_" + str(vcfStopCoordinate)]["rnaTumor"] # for each read, investigate the blat hits to see if this read is valid for (readId, blatHitList) in blatHitsDict.iteritems(): if (i_debug): logging.info("num of blat hits for read %s=%s", readId, len(blatHitList)) blatOverallReadDepth +=1 # find out if the read is valid or if it maps to other places in the genome if (i_blatOutputFormat == "PSL"): (isValidRead, validRead) = is_valid_read_psl_format(blatHitList, vcfChr, vcfStopCoordinate, i_debug) elif (i_blatOutputFormat == "BLAST"): (isValidRead, validRead) = is_valid_read_blast_format(blatHitList, vcfChr, vcfStopCoordinate, 0, i_debug) #(isValidRead, validRead) = is_valid_read_blast_format(blatHitList, vcfChr, vcfStopCoordinate, 1, i_debug) #(isValidRead, validRead) = is_valid_read_blast_format(blatHitList, vcfChr, vcfStopCoordinate, 2, i_debug) # if we have only one valid blat hit, then the read doesn't map to other places in the genome very well, so let's use it if (isValidRead): numValidReads += 1 if (i_debug): logging.debug("ValidRead: %s", validRead) if (blatOverallReadDepth > 0): altPercent = round(numValidReads/float(blatOverallReadDepth),2) else: altPercent = 0.0 if (numValidReads < i_readDepthCutoff or altPercent < i_readPercentCutoff): modTypeFilters[modType] = "blat" else: modTypeFilters[modType] = "PASS" atLeastOnePass = True if (i_debug): logging.info("blatOverallReadDepth=%s, numValidReads=%s, altPercent=%s", str(blatOverallReadDepth), str(numValidReads), str(altPercent)) logging.info("modType=%s, passed? %s", modType, modTypeFilters[modType]) logging.info("blatFilter originalDepth=%s, afterBlatDepth=%s", str(blatOverallReadDepth), str(numValidReads)) # make a copy of the list to manipulate modTypesTmpList = list(modTypes) modChanges = vcfInfoDict["MC"] # if at least one passed, then remove the ones that didn't for (modType, modChange) in izip(modTypes, modChanges): # if at least one passed, then remove the ones that didn't if (atLeastOnePass): if (modTypeFilters[modType] == "blat"): modTypesTmpList.remove(modType) modChanges.remove(modChange) # set the modTypes and modChanges vcfInfoDict["MT"] = modTypesTmpList vcfInfoDict["MC"] = modChanges # if at least one passed, then set pass if (atLeastOnePass): vcfFilterSet = ["PASS"] else: # if the user wants to keep the previous filters if (i_keepPreviousFiltersFlag): # if the call previous passed, then just set blat if (len(vcfFilterSet) == 1 and "PASS" in vcfFilterSet): vcfFilterSet = ["blat"] # otherwise, add it to the previous filters else: vcfFilterSet.add("blat") # otherwise, just set the blat filter else: vcfFilterSet = ["blat"] output = [vcfChr, str(vcfStopCoordinate), vcfId, vcfRef, vcfAlt, vcfScore, ";".join(vcfFilterSet)] # add the modified info dict infoField = "" for key in sorted(vcfInfoDict.iterkeys()): if (len(vcfInfoDict[key]) == 0): continue elif ("True" in vcfInfoDict[key]): infoField += key + ";" else: infoField += key + "=" + ",".join(vcfInfoDict[key]) + ";" output.append(infoField.rstrip(";")) output.append(restOfLine) if (i_outputFilename != None): i_outputFileHandler.write("\t".join(output) + "\n") else: print >> sys.stdout, "\t".join(output) stopTime = time.time() logging.info("Total time for Id %s: Total time=%s hrs, %s mins, %s secs", i_id, ((stopTime-startTime)/(3600)), ((stopTime-startTime)/60), (stopTime-startTime)) # close the files if (i_outputFilename != None): i_outputFileHandler.close() return
def main(): # python filterByPybed.py TCGA-AB-2995 12 ../data/test/filterBlacklist.bed # ../data/test/TCGA-AB-2995.vcf blck -d FILTER --includeOverlaps # --includeFilterName --log=DEBUG -f "##FILTER=<ID=blck,Description= # \"Position overlaps 1000 Genomes Project blacklist\">" # create the usage statement usage = "usage: python %prog id chrom filterFile vcfFile filterName [Opts]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option( "-f", "--filterHeader", dest="filterHeader", metavar="FILTER_HEADER", help="the INFO or FORMAT line to be included in the VCF header") i_cmdLineParser.add_option( "-p", "--includeOverlaps", action="store_true", default=False, dest="includeOverlaps", help="whether the events that overlap should be considered to PASS " + "(True) or FILTER (False), %default by default") i_cmdLineParser.add_option( "-n", "--includeFilterName", action="store_true", default=False, dest="includeFilterName", help="whether the filter name should be included in the INFO or " + "FILTER fields of the VCF output, %default by default") i_cmdLineParser.add_option( "-c", "--includeFilterCount", action="store_true", default=False, dest="includeFilterCount", help="whether the number of overlaps with the filters should be " + "included, %default by default") i_cmdLineParser.add_option( "-d", "--filterField", default="FILTER", dest="filterField", help="the column where the filter name should be included " + "(e.g. INFO or FILTER) in the VCF output, %default by default") i_cmdLineParser.add_option( "-s", "--idField", default="ID", dest="idField", help="the column where the id should be included " + "(e.g. ID or INFO) in the VCF output, %default by default") i_cmdLineParser.add_option( "-i", "--includeIdName", action="store_true", default=False, dest="includeIdName", help="whether the name found in the filtering file should " + "be included in the VCF output, %default by default") i_cmdLineParser.add_option( "-o", "--outputFilename", dest="outputFilename", metavar="OUTPUT_FILE", help="the name of the output file, sys.stdout by default") i_cmdLineParser.add_option( "-l", "--log", default="WARNING", dest="logLevel", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), " + "%default by default") i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") i_cmdLineParser.add_option( "-b", "--binSize", dest="binSize", default=int(10000), metavar="BIN_SIZE", help="the size of the interval between each bin, %default by default") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(5, 29, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = str(i_cmdLineArgs[0]) i_chr = str(i_cmdLineArgs[1]) i_filterFilename = str(i_cmdLineArgs[2]) i_vcfFilename = str(i_cmdLineArgs[3]) i_filterName = str(i_cmdLineArgs[4]) # get the optional params with default values i_includeOverlapsFlag = i_cmdLineOptions.includeOverlaps i_includeFilterName = i_cmdLineOptions.includeFilterName i_includeFilterCount = i_cmdLineOptions.includeFilterCount i_filterField = i_cmdLineOptions.filterField i_idField = i_cmdLineOptions.idField i_includeIdName = i_cmdLineOptions.includeIdName i_logLevel = i_cmdLineOptions.logLevel i_binSize = i_cmdLineOptions.binSize # try to get any optional parameters with no defaults i_outputFilename = None i_logFilename = None i_filterHeader = None if (i_cmdLineOptions.outputFilename is not None): i_outputFilename = str(i_cmdLineOptions.outputFilename) if (i_cmdLineOptions.logFilename is not None): i_logFilename = str(i_cmdLineOptions.logFilename) if (i_cmdLineOptions.filterHeader is not None): i_filterHeader = str(i_cmdLineOptions.filterHeader) # assuming logLevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the " + "following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename is not sys.stdout): logging.basicConfig( level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig( level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # do some debugging if (i_debug): logging.debug("id=%s", i_id) logging.debug("chr=%s", i_chr) logging.debug("filterFile=%s", i_filterFilename) logging.debug("vcfFile=%s", i_vcfFilename) logging.debug("output=%s", i_outputFilename) logging.debug("logLevel=%s", i_logLevel) logging.debug("logFile=%s", i_logFilename) logging.debug("filterName=%s", i_filterName) logging.debug("filterHeader=%s", i_filterHeader) logging.debug("includeOverlapsFlag=%s", i_includeOverlapsFlag) logging.debug("includeFilterName=%s", i_includeFilterName) logging.debug("includeFilterCount=%s", i_includeFilterCount) logging.debug("filterField=%s", i_filterField) logging.debug("idField=%s", i_idField) logging.debug("includeIdName=%s", i_includeIdName) logging.debug("binSize=%s", i_binSize) # check for any errors writeFilenameList = [] if (i_outputFilename is not None): writeFilenameList += [i_outputFilename] if (i_logFilename is not None): writeFilenameList += [i_logFilename] readFilenameList = [i_filterFilename, i_vcfFilename] if (not radiaUtil.check_for_argv_errors(None, readFilenameList, writeFilenameList)): sys.exit(1) filter_events(i_id, i_chr, i_filterFilename, i_vcfFilename, i_outputFilename, i_filterName, i_filterField, i_includeOverlapsFlag, i_includeFilterName, i_idField, i_includeIdName, i_includeFilterCount, i_filterHeader, i_binSize, i_debug) return
def main(): # command for running this on a small test case: #python filterByBlat.py TCGA-00-4454 7 ../data/test/TCGA-00-4454_EGFR.vcf ../data/test/TCGA-00-4454_EGFR.fa ../data/test/TCGA-00-4454_EGFR.psl --dnaNormalFilename=../data/test/TCGA-00-4454_EGFR.reads startTime = time.time() # create the usage statement usage = "usage: python %prog id chrom vcfFile blatInputFile blatOutputFile [Options]" i_cmdLineParser = OptionParser(usage=usage) # add the optional parameters i_cmdLineParser.add_option( "-c", "--allVCFCalls", action="store_false", default=True, dest="passedVCFCallsOnly", help= "by default only the VCF calls that have passed all filters thus far are processed, include this argument if all of the VCF calls should be processed" ) i_cmdLineParser.add_option( "-k", "--keepPreviousFilters", action="store_true", default=False, dest="keepPreviousFilters", help= "by default the previous filters are overwritten with the blat filter, include this argument if the previous filters should be kept" ) i_cmdLineParser.add_option( "-o", "--outputFilename", dest="outputFilename", metavar="OUTPUT_FILE", help="the name of the output file, STDOUT by default") i_cmdLineParser.add_option("-b", "--blatOutputFormat", dest="blatOutputFormat", metavar="OUTPUT_FORMAT", default="BLAST", help="the BLAT output format, BLAST by default") i_cmdLineParser.add_option( "-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help= "the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default" ) i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") i_cmdLineParser.add_option( "-n", "--blatDnaNormalReads", action="store_true", default=False, dest="blatDnaNormalReads", help="include this argument if the normal DNA reads should be processed" ) i_cmdLineParser.add_option( "-x", "--blatRnaNormalReads", action="store_true", default=False, dest="blatRnaNormalReads", help="include this argument if the normal RNA reads should be processed" ) i_cmdLineParser.add_option( "-t", "--blatDnaTumorReads", action="store_true", default=False, dest="blatDnaTumorReads", help="include this argument if the tumor DNA reads should be processed" ) i_cmdLineParser.add_option( "-r", "--blatRnaTumorReads", action="store_true", default=False, dest="blatRnaTumorReads", help="include this argument if the tumor RNA reads should be processed" ) i_cmdLineParser.add_option( "-d", "--readDepthCutoff", type="int", default=int(4), dest="readDepthCutoff", metavar="READ_DP_CUTOFF", help= "the minimum number of valid reads that are necessary, %default by default" ) i_cmdLineParser.add_option( "-p", "--readPercentCutoff", type="float", default=float(0.10), dest="readPercentCutoff", metavar="READ_PERCENT_CUTOFF", help= "the minimum percentage of valid reads that are necessary, %default by default" ) #i_cmdLineParser.add_option("-e", "--eValueCutoff", type="float", default=float(10e-6), dest="eValueCutoff", metavar="EVAL_CUTOFF", help="the e-value cutoff for determining if a blat hit is significant, %default by default") #i_cmdLineParser.add_option("-u", "--upperIdentityCutoff", type="float", default=float(0.95), dest="upperIdentityCutoff", metavar="UPPER_CUTOFF", help="the upper cutoff for the match length adjusted identity to determine if a blat hit is significant, %default by default") #i_cmdLineParser.add_option("-l", "--lowerIdentityCutoff", type="float", default=float(0.5), dest="lowerIdentityCutoff", metavar="LOWER_CUTOFF", help="the lower cutoff for the match length adjusted identity to determine if a second blat hit is significant, %default by default") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(5, 27, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = str(i_cmdLineArgs[0]) i_vcfFilename = str(i_cmdLineArgs[1]) i_blatInputFilename = str(i_cmdLineArgs[2]) i_blatOutputFilename = str(i_cmdLineArgs[3]) # get the optional params with default values i_passedVCFCallsOnlyFlag = i_cmdLineOptions.passedVCFCallsOnly i_keepPreviousFiltersFlag = i_cmdLineOptions.keepPreviousFilters i_blatOutputFormat = i_cmdLineOptions.blatOutputFormat i_logLevel = i_cmdLineOptions.logLevel i_readDepthCutoff = i_cmdLineOptions.readDepthCutoff i_readPercentCutoff = i_cmdLineOptions.readPercentCutoff #i_eValueCutoff = i_cmdLineOptions.eValueCutoff #i_upperIdentityCutoff = i_cmdLineOptions.upperIdentityCutoff #i_lowerIdentityCutoff = i_cmdLineOptions.lowerIdentityCutoff i_blatDnaNormalReads = i_cmdLineOptions.blatDnaNormalReads i_blatDnaTumorReads = i_cmdLineOptions.blatDnaTumorReads i_blatRnaNormalReads = i_cmdLineOptions.blatRnaNormalReads i_blatRnaTumorReads = i_cmdLineOptions.blatRnaTumorReads # try to get any optional parameters with no defaults i_outputFilename = None i_logFilename = None if (i_cmdLineOptions.outputFilename != None): i_outputFilename = str(i_cmdLineOptions.outputFilename) if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError( "Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("id=%s", i_id) logging.debug("vcfFilename=%s", i_vcfFilename) logging.debug("blatInputFilename=%s", i_blatInputFilename) logging.debug("blatOutputFilename=%s", i_blatOutputFilename) logging.debug("passedCallsOnly? %s", i_passedVCFCallsOnlyFlag) logging.debug("keepPreviousFiltersFlag? %s", i_keepPreviousFiltersFlag) logging.debug("blatOutputFormat=%s", i_blatOutputFormat) logging.debug("blatDnaNormal? %s", i_blatDnaNormalReads) logging.debug("blatDnaTumor? %s", i_blatDnaTumorReads) logging.debug("blatRnaNormal? %s", i_blatRnaNormalReads) logging.debug("blatRnaTumor? %s", i_blatRnaTumorReads) logging.debug("readDepthCutoff=%s", i_readDepthCutoff) logging.debug("readPerentCutoff=%s", i_readPercentCutoff) # check for any errors i_writeFilenameList = [] if (i_outputFilename != None): i_writeFilenameList = [i_outputFilename] if (i_logFilename != None): i_writeFilenameList = [i_logFilename] i_readFilenameList = [ i_vcfFilename, i_blatInputFilename, i_blatOutputFilename ] if (not radiaUtil.check_for_argv_errors(None, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # open the output stream i_outputFileHandler = None if (i_outputFilename != None): i_outputFileHandler = get_write_fileHandler(i_outputFilename) # get the BLAT results i_blatCoordinateDict = parse_blat_output(i_blatOutputFilename, i_blatOutputFormat, i_debug) # get the VCF generator i_vcfGenerator = get_vcf_data(i_vcfFilename, i_passedVCFCallsOnlyFlag, i_debug) for (vcfChr, vcfStopCoordinate, vcfId, vcfRef, vcfAlt, vcfScore, vcfFilterSet, vcfInfoDict, restOfLine) in i_vcfGenerator: if (i_debug): logging.debug("VCF Data: %s %s %s %s %s %s %s %s %s", vcfChr, str(vcfStopCoordinate), vcfId, vcfRef, vcfAlt, vcfScore, str(vcfFilterSet), str(vcfInfoDict), restOfLine) modTypes = vcfInfoDict["MT"] modTypeFilters = dict() atLeastOnePass = False for modType in modTypes: blatHitsDict = dict() blatOverallReadDepth = 0 numValidReads = 0 if (modType == "NOR_EDIT" and i_blatRnaNormalReads): if ("rnaNormal" in i_blatCoordinateDict[vcfChr + "_" + str(vcfStopCoordinate)]): # for each coordinate, get a dict of reads and corresponding blat hits blatHitsDict = i_blatCoordinateDict[ vcfChr + "_" + str(vcfStopCoordinate)]["rnaNormal"] elif ((modType == "SOM" or modType == "TUM_EDIT") and i_blatRnaTumorReads): if ("rnaTumor" in i_blatCoordinateDict[vcfChr + "_" + str(vcfStopCoordinate)]): # for each coordinate, get a dict of reads and corresponding blat hits blatHitsDict = i_blatCoordinateDict[ vcfChr + "_" + str(vcfStopCoordinate)]["rnaTumor"] # for each read, investigate the blat hits to see if this read is valid for (readId, blatHitList) in blatHitsDict.iteritems(): if (i_debug): logging.debug("num of blat hits for read %s=%s", readId, len(blatHitList)) blatOverallReadDepth += 1 # find out if the read is valid or if it maps to other places in the genome if (i_blatOutputFormat == "PSL"): (isValidRead, validRead) = is_valid_read_psl_format( blatHitList, vcfChr, vcfStopCoordinate, i_debug) elif (i_blatOutputFormat == "BLAST"): (isValidRead, validRead) = is_valid_read_blast_format( blatHitList, vcfChr, vcfStopCoordinate, 0, i_debug) #(isValidRead, validRead) = is_valid_read_blast_format(blatHitList, vcfChr, vcfStopCoordinate, 1, i_debug) #(isValidRead, validRead) = is_valid_read_blast_format(blatHitList, vcfChr, vcfStopCoordinate, 2, i_debug) # if we have only one valid blat hit, then the read doesn't map to other places in the genome very well, so let's use it if (isValidRead): numValidReads += 1 if (i_debug): logging.debug("ValidRead: %s", validRead) if (blatOverallReadDepth > 0): altPercent = round(numValidReads / float(blatOverallReadDepth), 2) else: altPercent = 0.0 if (numValidReads < i_readDepthCutoff or altPercent < i_readPercentCutoff): modTypeFilters[modType] = "blat" else: modTypeFilters[modType] = "PASS" atLeastOnePass = True if (i_debug): logging.debug( "blatOverallReadDepth=%s, numValidReads=%s, altPercent=%s", str(blatOverallReadDepth), str(numValidReads), str(altPercent)) logging.debug("modType=%s, passed? %s", modType, modTypeFilters[modType]) logging.debug("blatFilter originalDepth=%s, afterBlatDepth=%s", str(blatOverallReadDepth), str(numValidReads)) # make a copy of the list to manipulate modTypesTmpList = list(modTypes) modChanges = vcfInfoDict["MC"] # if at least one passed, then remove the ones that didn't for (modType, modChange) in izip(modTypes, modChanges): # if at least one passed, then remove the ones that didn't if (atLeastOnePass): if (modTypeFilters[modType] == "blat"): modTypesTmpList.remove(modType) modChanges.remove(modChange) # set the modTypes and modChanges vcfInfoDict["MT"] = modTypesTmpList vcfInfoDict["MC"] = modChanges # if at least one passed, then set pass if (atLeastOnePass): vcfFilterSet = ["PASS"] else: # if the user wants to keep the previous filters if (i_keepPreviousFiltersFlag): # if the call previous passed, then just set blat if (len(vcfFilterSet) == 1 and "PASS" in vcfFilterSet): vcfFilterSet = ["blat"] # otherwise, add it to the previous filters else: vcfFilterSet.add("blat") # otherwise, just set the blat filter else: vcfFilterSet = ["blat"] # update the mod filters modTypes = vcfInfoDict["MT"] modChanges = vcfInfoDict["MC"] origins = vcfInfoDict["ORIGIN"] modFilters = [] if vcfInfoDict["MF"] is None else vcfInfoDict["MF"] modFilterTypes = [] if vcfInfoDict["MFT"] is None else vcfInfoDict[ "MFT"] for origin in origins: for (modType, modChange) in izip(modTypes, modChanges): modFilterTypes.append("_".join( [origin, modType, modChange])) modFilters.append("_".join(vcfFilterSet)) vcfInfoDict["MF"] = modFilters vcfInfoDict["MFT"] = modFilterTypes output = [ vcfChr, str(vcfStopCoordinate), vcfId, vcfRef, vcfAlt, vcfScore, ";".join(vcfFilterSet) ] # add the modified info dict infoField = "" for key in sorted(vcfInfoDict.iterkeys()): if (len(vcfInfoDict[key]) == 0): continue elif ("True" in vcfInfoDict[key]): infoField += key + ";" else: infoField += key + "=" + ",".join(vcfInfoDict[key]) + ";" output.append(infoField.rstrip(";")) output.append(restOfLine) if (i_outputFilename != None): i_outputFileHandler.write("\t".join(output) + "\n") else: print >> sys.stdout, "\t".join(output) stopTime = time.time() logging.info("Total time for Id %s: Total time=%s hrs, %s mins, %s secs", i_id, ((stopTime - startTime) / (3600)), ((stopTime - startTime) / 60), (stopTime - startTime)) # close the files if (i_outputFilename != None): i_outputFileHandler.close() return
def main(): startTime = time.time() # create the usage statement usage = "usage: python %prog passingFile originalFile outputFile [Options]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option( "-l", "--log", default="WARNING", dest="logLevel", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), " + "%default by default") i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDERR by default") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(3, 10, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_passingFilename = i_cmdLineArgs[0] i_originalFilename = i_cmdLineArgs[1] i_outputFilename = i_cmdLineArgs[2] # get the optional params with default values i_logLevel = i_cmdLineOptions.logLevel i_logFilename = None if (i_cmdLineOptions.logFilename is not None): i_logFilename = str(i_cmdLineOptions.logFilename) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the " + "following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename is not None): logging.basicConfig( level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig( level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("passingFile=%s", i_passingFilename) logging.debug("originalFile=%s", i_originalFilename) logging.debug("outputFilename=%s", i_outputFilename) # check for any errors i_readFilenameList = [i_passingFilename, i_originalFilename] i_writeFilenameList = [i_outputFilename] i_dirList = None if (not radiaUtil.check_for_argv_errors(i_dirList, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # get the VCF generator (passHeaderList, chromLine, passInfoList, passFilterList, passCoordinateDict) = get_vcf_data(i_passingFilename, i_debug) (orgHeaderList, chromLine, orgInfoList, orgFilterList, orgCoordinateDict) = get_vcf_data(i_originalFilename, i_debug) outputFileHandler = radiaUtil.get_write_fileHandler(i_outputFilename) for headerLine in orgHeaderList: outputFileHandler.write(headerLine + "\n") for headerLine in orgInfoList: outputFileHandler.write(headerLine + "\n") for headerLine in passInfoList: if (headerLine not in orgInfoList): outputFileHandler.write(headerLine + "\n") for headerLine in orgFilterList: outputFileHandler.write(headerLine + "\n") for headerLine in passFilterList: if (headerLine not in orgFilterList): outputFileHandler.write(headerLine + "\n") outputFileHandler.write(chromLine + "\n") numericKeys = orgCoordinateDict.keys() numericKeys.sort(key=int) for coordinate in numericKeys: if (coordinate in passCoordinateDict): line = passCoordinateDict[coordinate] else: line = orgCoordinateDict[coordinate] outputFileHandler.write(line) stopTime = time.time() logging.info("Total time=%s hrs, %s mins, %s secs", ((stopTime-startTime)/(3600)), ((stopTime-startTime)/60), (stopTime-startTime)) # close the files outputFileHandler.close() return
def main(): # command for running this on a small test case: #python createBlatFile.py TCGA-00-4454 7 ../data/test/TCGA-00-4454_EGFR.vcf ../data/test/tmp/ --dnaNormalFilename=../data/test/TCGA-00-4454_EGFR.reads startTime = time.time() # create the usage statement usage = "usage: python %prog id vcfFile headerFile [Options]" i_cmdLineParser = OptionParser(usage=usage) # add the optional parameters i_cmdLineParser.add_option("-c", "--allVCFCalls", action="store_false", default=True, dest="passedVCFCallsOnly", help="by default only the VCF calls that have passed all filters thus far are processed, include this argument if all of the VCF calls should be processed") i_cmdLineParser.add_option("-b", "--allReadBases", action="store_false", default=True, dest="altBasesOnly", help="by default only the reads with the alternate base are processed, include this argument if all of the reads should be processed") i_cmdLineParser.add_option("-d", "--maxReadDepth", type="int", default=int(8000), dest="maxReadDepth", metavar="MAX_READ_DEPTH", help="the maximum read depth to process from the samtools view command, %default by default") i_cmdLineParser.add_option("-o", "--outputFilename", dest="outputFilename", metavar="OUTPUT_FILE", help="the name of the output file, STDOUT by default") i_cmdLineParser.add_option("-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default") i_cmdLineParser.add_option("-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") i_cmdLineParser.add_option("", "--transcriptNameTag", dest="transcriptNameTag", help="the INFO key where the original transcript name can be found") i_cmdLineParser.add_option("", "--transcriptCoordinateTag", dest="transcriptCoordinateTag", help="the INFO key where the original transcript coordinate can be found") i_cmdLineParser.add_option("", "--transcriptStrandTag", dest="transcriptStrandTag", help="the INFO key where the original transcript strand can be found") i_cmdLineParser.add_option("", "--rnaIncludeSecondaryAlignments", action="store_true", default=False, dest="rnaIncludeSecondaryAlignments", help="if you align the RNA to transcript isoforms, then you may want to include RNA secondary alignments in the samtools mpileups") i_cmdLineParser.add_option("-n", "--blatDnaNormalReads", action="store_true", default=False, dest="blatDnaNormalReads", help="include this argument if the normal DNA reads should be processed") i_cmdLineParser.add_option("-x", "--blatRnaNormalReads", action="store_true", default=False, dest="blatRnaNormalReads", help="include this argument if the normal RNA reads should be processed") i_cmdLineParser.add_option("-t", "--blatDnaTumorReads", action="store_true", default=False, dest="blatDnaTumorReads", help="include this argument if the tumor DNA reads should be processed") i_cmdLineParser.add_option("-r", "--blatRnaTumorReads", action="store_true", default=False, dest="blatRnaTumorReads", help="include this argument if the tumor RNA reads should be processed") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(3,22,1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = i_cmdLineArgs[0] i_vcfFilename = i_cmdLineArgs[1] i_headerFilename = i_cmdLineArgs[2] # get the optional params with default values i_passedVCFCallsOnlyFlag = i_cmdLineOptions.passedVCFCallsOnly i_altBasesOnlyFlag = i_cmdLineOptions.altBasesOnly i_maxReadDepth = i_cmdLineOptions.maxReadDepth i_logLevel = i_cmdLineOptions.logLevel i_rnaIncludeSecondaryAlignments = i_cmdLineOptions.rnaIncludeSecondaryAlignments i_blatDnaNormalReads = i_cmdLineOptions.blatDnaNormalReads i_blatDnaTumorReads = i_cmdLineOptions.blatDnaTumorReads i_blatRnaNormalReads = i_cmdLineOptions.blatRnaNormalReads i_blatRnaTumorReads = i_cmdLineOptions.blatRnaTumorReads # try to get any optional parameters with no defaults i_readFilenameList = [i_vcfFilename, i_headerFilename] i_writeFilenameList = [] i_logFilename = None i_outputFilename = None i_transcriptNameTag = None i_transcriptCoordinateTag = None i_transcriptStrandTag = None if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) i_writeFilenameList += [i_logFilename] if (i_cmdLineOptions.outputFilename != None): i_outputFilename = str(i_cmdLineOptions.outputFilename) i_writeFilenameList += [i_outputFilename] if (i_cmdLineOptions.transcriptNameTag != None): i_transcriptNameTag = i_cmdLineOptions.transcriptNameTag if (i_cmdLineOptions.transcriptCoordinateTag != None): i_transcriptCoordinateTag = i_cmdLineOptions.transcriptCoordinateTag if (i_cmdLineOptions.transcriptStrandTag != None): i_transcriptStrandTag = i_cmdLineOptions.transcriptStrandTag # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug flag i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("id=%s", i_id) logging.debug("vcfFilename=%s", i_vcfFilename) logging.debug("headerFilename=%s", i_headerFilename) logging.debug("outputFilename=%s", i_outputFilename) logging.debug("passedCallsOnly? %s", i_passedVCFCallsOnlyFlag) logging.debug("altBasesOnlyFlag? %s", i_altBasesOnlyFlag) logging.debug("maxReadDepth %s", i_maxReadDepth) logging.debug("transcriptNameTag %s", i_transcriptNameTag) logging.debug("transcriptCoordinateTag %s", i_transcriptCoordinateTag) logging.debug("transcriptStrandTag %s", i_transcriptStrandTag) logging.debug("rnaIncludeSecondaryAlignments=%s" % i_rnaIncludeSecondaryAlignments) logging.debug("blatDnaNormal? %s", i_blatDnaNormalReads) logging.debug("blatDnaTumor? %s", i_blatDnaTumorReads) logging.debug("blatRnaNormal? %s", i_blatRnaNormalReads) logging.debug("blatRnaTumor? %s", i_blatRnaTumorReads) if (not radiaUtil.check_for_argv_errors(None, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # open the output stream i_outputFileHandler = None if (i_outputFilename != None): i_outputFileHandler = radiaUtil.get_write_fileHandler(i_outputFilename) # get the VCF generator i_vcfGenerator = get_vcf_data(i_vcfFilename, i_headerFilename, i_passedVCFCallsOnlyFlag, i_debug) # for each VCF call that should be investigated for (vcfChr, vcfStopCoordinate, vcfId, vcfRef, vcfAlt, vcfScore, vcfFilterSet, vcfInfoDict, restOfLine, vcfParamsDict) in i_vcfGenerator: if (i_debug): logging.debug("VCF Data: %s %s %s %s %s %s %s %s %s", vcfChr, str(vcfStopCoordinate), vcfId, vcfRef, vcfAlt, vcfScore, str(vcfFilterSet), str(vcfInfoDict), restOfLine) modTypes = vcfInfoDict["MT"] for modType in modTypes: # get the reads contributing to a call and put them in a blat query file if (i_blatDnaNormalReads): write_to_blat_file(i_outputFileHandler, vcfChr, vcfStopCoordinate, [vcfChr], [vcfStopCoordinate], [None], vcfParamsDict, vcfInfoDict, "dnaNormal", i_altBasesOnlyFlag, False, i_maxReadDepth, i_debug) if (modType == "NOR_EDIT" and i_blatRnaNormalReads): # if we should process the transcripts if ((i_transcriptNameTag != None) and (i_transcriptNameTag in vcfInfoDict)): write_to_blat_file(i_outputFileHandler, vcfChr, vcfStopCoordinate, vcfInfoDict[i_transcriptNameTag], vcfInfoDict[i_transcriptCoordinateTag], vcfInfoDict[i_transcriptStrandTag], vcfParamsDict, vcfInfoDict, "rnaNormal", i_altBasesOnlyFlag, i_rnaIncludeSecondaryAlignments, i_maxReadDepth, i_debug) else: write_to_blat_file(i_outputFileHandler, vcfChr, vcfStopCoordinate, [vcfChr], [vcfStopCoordinate], [None], vcfParamsDict, vcfInfoDict, "rnaNormal", i_altBasesOnlyFlag, i_rnaIncludeSecondaryAlignments, i_maxReadDepth, i_debug) if (i_blatDnaTumorReads): write_to_blat_file(i_outputFileHandler, vcfChr, vcfStopCoordinate, [vcfChr], [vcfStopCoordinate], [None], vcfParamsDict, vcfInfoDict, "dnaTumor", i_altBasesOnlyFlag, False, i_maxReadDepth, i_debug) if ((modType == "SOM" or modType == "TUM_EDIT") and i_blatRnaTumorReads): # if we should process the transcripts if ((i_transcriptNameTag != None) and (i_transcriptNameTag in vcfInfoDict)): write_to_blat_file(i_outputFileHandler, vcfChr, vcfStopCoordinate, list(vcfInfoDict[i_transcriptNameTag]), vcfInfoDict[i_transcriptCoordinateTag], vcfInfoDict[i_transcriptStrandTag], vcfParamsDict, vcfInfoDict, "rnaTumor", i_altBasesOnlyFlag, i_rnaIncludeSecondaryAlignments, i_maxReadDepth, i_debug) else: write_to_blat_file(i_outputFileHandler, vcfChr, vcfStopCoordinate, [vcfChr], [vcfStopCoordinate], [None], vcfParamsDict, vcfInfoDict, "rnaTumor", i_altBasesOnlyFlag, i_rnaIncludeSecondaryAlignments, i_maxReadDepth, i_debug) stopTime = time.time() logging.info("createBlatFile.py Id %s: Total time=%s hrs, %s mins, %s secs", i_id, ((stopTime-startTime)/(3600)), ((stopTime-startTime)/60), (stopTime-startTime)) # close the files if (i_outputFilename != None): i_outputFileHandler.close() return
def main(): #python filterByPybed.py TCGA-AB-2995 12 ../data/test/filterBlacklist.bed ../data/test/TCGA-AB-2995.vcf blck -d FILTER --includeOverlaps --includeFilterName --log=DEBUG -f "##FILTER=<ID=blck,Description=\"Position overlaps 1000 Genomes Project blacklist\">" # create the usage statement usage = "usage: python %prog id chrom filterFile vcfFile filterName [Options]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option( "-f", "--filterHeader", dest="filterHeader", metavar="FILTER_HEADER", help="the INFO or FORMAT line that should be included in the VCF header" ) i_cmdLineParser.add_option( "-p", "--includeOverlaps", action="store_true", default=False, dest="includeOverlaps", help= "whether the events that overlap should be considered to PASS (True) or FILTER (False), %default by default" ) i_cmdLineParser.add_option( "-n", "--includeFilterName", action="store_true", default=False, dest="includeFilterName", help= "whether the filter name should be included in the INFO or FILTER fields of the VCF output, %default by default" ) i_cmdLineParser.add_option( "-c", "--includeFilterCount", action="store_true", default=False, dest="includeFilterCount", help= "whether the number of overlaps with the filters should be included, %default by default" ) i_cmdLineParser.add_option( "-d", "--filterField", default="FILTER", dest="filterField", help= "the column where the filter name should be included (INFO or FILTER) field of the VCF output, %default by default" ) i_cmdLineParser.add_option( "-i", "--includeIdName", action="store_true", default=False, dest="includeIdName", help= "whether the name found in the filtering file should be included in the ID field of the VCF output, %default by default" ) i_cmdLineParser.add_option( "-o", "--outputFilename", dest="outputFilename", metavar="OUTPUT_FILE", help="the name of the output file, sys.stdout by default") i_cmdLineParser.add_option( "-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help= "the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default" ) i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") i_cmdLineParser.add_option( "-b", "--binSize", dest="binSize", default=int(10000), metavar="BIN_SIZE", help="the size of the interval between each bin, %default by default") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(5, 27, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = str(i_cmdLineArgs[0]) i_chr = str(i_cmdLineArgs[1]) i_filterFilename = str(i_cmdLineArgs[2]) i_vcfFilename = str(i_cmdLineArgs[3]) i_filterName = str(i_cmdLineArgs[4]) # get the optional params with default values i_includeOverlapsFlag = i_cmdLineOptions.includeOverlaps i_includeFilterName = i_cmdLineOptions.includeFilterName i_includeFilterCount = i_cmdLineOptions.includeFilterCount i_filterField = i_cmdLineOptions.filterField i_includeIdName = i_cmdLineOptions.includeIdName i_logLevel = i_cmdLineOptions.logLevel i_binSize = i_cmdLineOptions.binSize # try to get any optional parameters with no defaults i_outputFilename = None i_logFilename = None i_filterHeader = None if (i_cmdLineOptions.outputFilename != None): i_outputFilename = str(i_cmdLineOptions.outputFilename) if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) if (i_cmdLineOptions.filterHeader != None): i_filterHeader = str(i_cmdLineOptions.filterHeader) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError( "Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # do some debugging if (i_debug): logging.debug("id=%s", i_id) logging.debug("chr=%s", i_chr) logging.debug("filterFile=%s", i_filterFilename) logging.debug("vcfFile=%s", i_vcfFilename) logging.debug("output=%s", i_outputFilename) logging.debug("logLevel=%s", i_logLevel) logging.debug("logFile=%s", i_logFilename) logging.debug("filterName=%s", i_filterName) logging.debug("filterHeader=%s", i_filterHeader) logging.debug("includeOverlapsFlag=%s", i_includeOverlapsFlag) logging.debug("includeFilterName=%s", i_includeFilterName) logging.debug("includeFilterCount=%s", i_includeFilterCount) logging.debug("filterField=%s", i_filterField) logging.debug("includeIdName=%s", i_includeIdName) logging.debug("binSize=%s", i_binSize) # check for any errors writeFilenameList = [] if (i_outputFilename != None): writeFilenameList += [i_outputFilename] if (i_logFilename != None): writeFilenameList += [i_logFilename] readFilenameList = [i_filterFilename, i_vcfFilename] if (not radiaUtil.check_for_argv_errors(None, readFilenameList, writeFilenameList)): sys.exit(1) filter_events(i_id, i_chr, i_filterFilename, i_vcfFilename, i_outputFilename, i_filterName, i_filterField, i_includeOverlapsFlag, i_includeFilterName, i_includeIdName, i_includeFilterCount, i_filterHeader, i_binSize, i_debug) return
def main(): # create the usage statement usage = "usage: python %prog vcfFile rnaGeneFile rnaGeneFamilyFile [Opts]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option( "-o", "--outputFilename", default=sys.stdout, dest="outputFilename", metavar="OUTPUT_FILE", help="the name of the output file, STDOUT by default") i_cmdLineParser.add_option( "-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), " + "%default by default") i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") i_cmdLineParser.add_option( "-c", "--allVCFCalls", action="store_false", default=True, dest="passedVCFCallsOnly", help="by default only the VCF calls that have passed all filters " + "thus far are processed, include this argument if all of the " + "VCF calls should be processed") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(3, 14, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_vcfFilename = str(i_cmdLineArgs[0]) i_rnaGeneFilename = str(i_cmdLineArgs[1]) i_rnaGeneFamilyFilename = str(i_cmdLineArgs[2]) # get the optional params with default values i_logLevel = i_cmdLineOptions.logLevel i_passedVCFCallsOnlyFlag = i_cmdLineOptions.passedVCFCallsOnly # try to get any optional parameters with no defaults i_outputFilename = None i_logFilename = None if (i_cmdLineOptions.outputFilename is not None): i_outputFilename = str(i_cmdLineOptions.outputFilename) if (i_cmdLineOptions.logFilename is not None): i_logFilename = str(i_cmdLineOptions.logFilename) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the " + "following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename is not sys.stdout): logging.basicConfig( level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig( level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("vcfFilename=%s", i_vcfFilename) logging.debug("rnaGeneFilename=%s", i_rnaGeneFilename) logging.debug("rnaGeneFamilyFilename=%s", i_rnaGeneFamilyFilename) logging.debug("outputFilename=%s", i_outputFilename) logging.debug("logFilename=%s", i_logFilename) logging.debug("passedOnly?=%s", i_passedVCFCallsOnlyFlag) # check for any errors i_writeFilenameList = [] if (i_outputFilename is not sys.stdout): i_writeFilenameList = [i_outputFilename] if (i_logFilename is not None): i_writeFilenameList = [i_logFilename] i_readFilenameList = [i_vcfFilename, i_rnaGeneFilename, i_rnaGeneFamilyFilename] if (not radiaUtil.check_for_argv_errors(None, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # open the input stream i_vcfFileHandler = radiaUtil.get_read_fileHandler(i_vcfFilename) # open the output stream if i_outputFilename is not sys.stdout: i_outputFileHandler = radiaUtil.get_write_fileHandler(i_outputFilename) else: i_outputFileHandler = i_outputFilename # get the RNA gene blacklists (i_rnaGeneList, i_rnaGeneFamilyList) = get_rna_genes(i_rnaGeneFilename, i_rnaGeneFamilyFilename, i_debug) hasAddedFilterHeader = False for line in i_vcfFileHandler: if (i_debug): logging.debug("vcfLine: %s", line) # if it is an empty line, then just continue if (line.isspace()): continue # if we find the FILTER section, then add the filters from here elif ((not hasAddedFilterHeader) and (line.startswith("##FILTER"))): hasAddedFilterHeader = True i_outputFileHandler.write( "##FILTER=<ID=rgene,Description=\"This gene is on the " + "RNA gene blacklist\">\n") i_outputFileHandler.write( "##FILTER=<ID=rgfam,Description=\"This gene family is on " + "the RNA gene family blacklist\">\n") i_outputFileHandler.write(line) # these lines are from previous scripts in the pipeline, so output them elif (line.startswith("#")): i_outputFileHandler.write(line) # if we are only suppose to process the passed calls # and this call has not passed, then skip it elif (i_passedVCFCallsOnlyFlag and "PASS" not in line): i_outputFileHandler.write(line) # now we are to the data else: # strip the carriage return and newline characters line = line.rstrip("\r\n") # split the line on the tab splitLine = line.split("\t") filterSet = set(splitLine[6].split(";")) # if there are no filters so far, then clear the list if (len(filterSet) == 1 and "PASS" in filterSet): filterSet = set() # parse the info column and create a dict infoList = splitLine[7].split(";") infoDict = collections.defaultdict(list) for info in infoList: keyValueList = info.split("=") # some keys are just singular without a value (e.g. DB, etc.) if (len(keyValueList) == 1): infoDict[keyValueList[0]] = ["True"] else: # the value can be a comma separated list infoDict[keyValueList[0]] = keyValueList[1].split(",") effectList = infoDict["EFF"] effectRegEx = re.compile("(\\w).*\\({1}") ignoreEffectsList = ["UPSTREAM", "DOWNSTREAM"] isRnaBlacklistGene = False isRnaBlacklistGeneFamily = False for rawEffect in effectList: rawEffect = rawEffect.rstrip(")") iterator = effectRegEx.finditer(rawEffect) # for each match object in the iterator for match in iterator: effect = match.group() rawEffect = rawEffect.replace(effect, "") effect = effect.rstrip("(") if (effect in ignoreEffectsList): continue effectParts = rawEffect.split("|") # effectImpact = effectParts[0] # functionalClass = effectParts[1] # codonChange = effectParts[2] # aaChange = effectParts[3] # aaLength = effectParts[4] geneName = effectParts[5] transcriptBiotype = effectParts[6] # geneCoding = effectParts[7] # ensembleId = effectParts[8] # exonNumber = effectParts[9] # genotypeNumber = effectParts[10] # the RNA gene list can have "RP11" and that # should filter out any gene with RP11 in it for rnaGene in i_rnaGeneList: if (rnaGene in geneName): isRnaBlacklistGene = True break if (transcriptBiotype in i_rnaGeneFamilyList): isRnaBlacklistGeneFamily = True output = ["\t".join(splitLine[0:6])] # if the filter should be applied if (isRnaBlacklistGene): filterSet.add("rgene") # if the filter should be applied if (isRnaBlacklistGeneFamily): filterSet.add("rgfam") # if there are no filters so far, then this call passes if (len(filterSet) == 0): filterSet.add("PASS") output.append(";".join(filterSet)) output.append("\t".join(splitLine[7:])) if (i_outputFilename is not sys.stdout): i_outputFileHandler.write("\t".join(output) + "\n") else: print >> sys.stdout, "\t".join(output) # close the files i_vcfFileHandler.close() if (i_outputFilename is not sys.stdout): i_outputFileHandler.close() return
def main(): # command for running this on a small test case: # python filterByBlat.py TCGA-00-4454 7 ../data/test/TCGA-00-4454_EGFR.vcf # ../data/test/TCGA-00-4454_EGFR.fa ../data/test/TCGA-00-4454_EGFR.psl # --dnaNormalFilename=../data/test/TCGA-00-4454_EGFR.reads startTime = time.time() # create the usage statement usage = "usage: python %prog id chrom vcfFile blatOutputFile [Options]" i_cmdLineParser = OptionParser(usage=usage) # add the optional parameters i_cmdLineParser.add_option( "-c", "--allVCFCalls", action="store_false", default=True, dest="passedVCFCallsOnly", help="by default only the VCF calls that have passed all filters " + "thus far are processed, include this argument if all of the " + "VCF calls should be processed") i_cmdLineParser.add_option( "-k", "--keepPreviousFilters", action="store_true", default=False, dest="keepPreviousFilters", help="by default the previous filters are overwritten with the blat " + "filter, include this argument if the previous filters should " + "be kept") i_cmdLineParser.add_option( "-o", "--outputFilename", dest="outputFilename", metavar="OUTPUT_FILE", default=sys.stdout, help="the name of the output file, STDOUT by default") i_cmdLineParser.add_option("-b", "--blatOutputFormat", dest="blatOutputFormat", metavar="OUTPUT_FORMAT", default="BLAST", help="the BLAT output format, BLAST by default") i_cmdLineParser.add_option( "-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), " + "%default by default") i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDERR by default") i_cmdLineParser.add_option( "", "--transcriptNameTag", dest="transcriptNameTag", help="the INFO key where the original transcript name can be found") i_cmdLineParser.add_option( "", "--transcriptCoordinateTag", dest="transcriptCoordinateTag", help="the INFO key where the original transcript" + "coordinate can be found") i_cmdLineParser.add_option( "", "--transcriptStrandTag", dest="transcriptStrandTag", help="the INFO key where the original transcript strand can be found") i_cmdLineParser.add_option( "", "--rnaIncludeSecondaryAlignments", action="store_true", default=False, dest="rnaIncludeSecondaryAlignments", help="if you align the RNA to transcript isoforms, then you may " + "want to include RNA secondary alignments in the pileup") i_cmdLineParser.add_option( "-n", "--blatDnaNormalReads", action="store_true", default=False, dest="blatDnaNormalReads", help="include this argument if the normal DNA reads " + "should be processed") i_cmdLineParser.add_option( "-x", "--blatRnaNormalReads", action="store_true", default=False, dest="blatRnaNormalReads", help="include this argument if the normal RNA reads " + "should be processed") i_cmdLineParser.add_option( "-t", "--blatDnaTumorReads", action="store_true", default=False, dest="blatDnaTumorReads", help="include this argument if the tumor DNA reads " + "should be processed") i_cmdLineParser.add_option( "-r", "--blatRnaTumorReads", action="store_true", default=False, dest="blatRnaTumorReads", help="include this argument if the tumor RNA reads " + "should be processed") i_cmdLineParser.add_option( "-d", "--minReadDepth", type="int", default=int(4), dest="minReadDepth", metavar="MIN_READ_DP", help="the minimum number of valid reads that are necessary, " + "%default by default") i_cmdLineParser.add_option( "-p", "--minReadPercent", type="float", default=float(0.10), dest="minReadPercent", metavar="MIN_READ_PCT", help="the minimum percentage of valid reads that are necessary, " + "%default by default") i_cmdLineParser.add_option( "-m", "--minOrderMagnitude", type="int", default=float(0), dest="minOrderMagnitude", metavar="MIN_ORDER_MAGNITUDE", help="the minimum order of magnitude difference between the blat " + "hit at the query position vs. the next best blat hit in order " + "for the read to be valid, %default by default") ''' i_cmdLineParser.add_option( "-e", "--minEValue", type="float", default=float(10e-6), dest="minEValue", metavar="MIN_EVALUE", help="the minimum e-value needed for a blat hit to be significant, " + "%default by default") i_cmdLineParser.add_option( "-u", "--maxIdentity", type="float", default=float(0.95), dest="maxIdentity", metavar="MAX_IDENTITY", help="the maximum match length adjusted identity for a blat hit to " + "be significant, %default by default") i_cmdLineParser.add_option( "-l", "--minIdentity", type="float", default=float(0.5), dest="minIdentity", metavar="MIN_IDENTITY", help="the minimum match length adjusted identity for a blat hit to " + "be significant, %default by default") ''' # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(5, 27, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (cmdLineOpts, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = str(i_cmdLineArgs[0]) i_vcfFilename = str(i_cmdLineArgs[1]) i_blatOutputFilename = str(i_cmdLineArgs[2]) # get the optional params with default values i_passedVCFCallsOnlyFlag = cmdLineOpts.passedVCFCallsOnly i_keepPreviousFiltersFlag = cmdLineOpts.keepPreviousFilters i_blatOutputFormat = cmdLineOpts.blatOutputFormat i_logLevel = cmdLineOpts.logLevel i_rnaIncludeSecondaryAlignments = cmdLineOpts.rnaIncludeSecondaryAlignments i_minReadDepth = cmdLineOpts.minReadDepth i_minReadPercent = cmdLineOpts.minReadPercent i_minOrderMagnitude = cmdLineOpts.minOrderMagnitude # i_minEValue = cmdLineOpts.minEValue # i_maxIdentity = cmdLineOpts.maxIdentity # i_minIdentity = cmdLineOpts.minIdentity i_blatDnaNormalReads = cmdLineOpts.blatDnaNormalReads i_blatDnaTumorReads = cmdLineOpts.blatDnaTumorReads i_blatRnaNormalReads = cmdLineOpts.blatRnaNormalReads i_blatRnaTumorReads = cmdLineOpts.blatRnaTumorReads # try to get any optional parameters with no defaults i_outputFilename = None i_logFilename = None i_transcriptNameTag = None i_transcriptCoordinateTag = None i_transcriptStrandTag = None if (cmdLineOpts.outputFilename is not None): i_outputFilename = cmdLineOpts.outputFilename if (cmdLineOpts.logFilename is not None): i_logFilename = cmdLineOpts.logFilename if (cmdLineOpts.transcriptNameTag is not None): i_transcriptNameTag = cmdLineOpts.transcriptNameTag if (cmdLineOpts.transcriptCoordinateTag is not None): i_transcriptCoordinateTag = cmdLineOpts.transcriptCoordinateTag if (cmdLineOpts.transcriptStrandTag is not None): i_transcriptStrandTag = cmdLineOpts.transcriptStrandTag # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError( "Invalid log level: '%s' must be one of the " + "following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename is not None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("id=%s", i_id) logging.debug("vcfFilename=%s", i_vcfFilename) logging.debug("blatOutputFilename=%s", i_blatOutputFilename) logging.debug("passedCallsOnly? %s", i_passedVCFCallsOnlyFlag) logging.debug("keepPreviousFiltersFlag? %s", i_keepPreviousFiltersFlag) logging.debug("blatOutputFormat=%s", i_blatOutputFormat) logging.debug("transcriptNameTag %s", i_transcriptNameTag) logging.debug("transcriptCoordinateTag %s", i_transcriptCoordinateTag) logging.debug("transcriptStrandTag %s", i_transcriptStrandTag) logging.debug("rnaInclSecAlign=%s" % i_rnaIncludeSecondaryAlignments) logging.debug("blatDnaNormal? %s", i_blatDnaNormalReads) logging.debug("blatDnaTumor? %s", i_blatDnaTumorReads) logging.debug("blatRnaNormal? %s", i_blatRnaNormalReads) logging.debug("blatRnaTumor? %s", i_blatRnaTumorReads) logging.debug("minReadDepth=%s", i_minReadDepth) logging.debug("minReadPercent=%s", i_minReadPercent) logging.debug("minOrderMagnitude=%s", i_minOrderMagnitude) # check for any errors i_writeFilenameList = [] if (cmdLineOpts.outputFilename is not sys.stdout): i_writeFilenameList = [i_outputFilename] if (i_logFilename is not None): i_writeFilenameList = [i_logFilename] i_readFilenameList = [i_vcfFilename, i_blatOutputFilename] if (not radiaUtil.check_for_argv_errors(None, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # open the output stream if i_outputFilename is not sys.stdout: i_outputFileHandler = radiaUtil.get_write_fileHandler(i_outputFilename) else: i_outputFileHandler = i_outputFilename # get the VCF generator i_vcfGenerator = get_vcf_data(i_vcfFilename, i_passedVCFCallsOnlyFlag, i_debug) # get the blat hits generator i_blatGenerator = parse_blat_output(i_blatOutputFilename, i_blatOutputFormat, i_debug) for (vcfLine, blatHitsDict) in izip(i_vcfGenerator, i_blatGenerator): if (i_debug): logging.debug("VCF Line=%s", vcfLine) logging.debug("Len Blat Hits=%s", len(blatHitsDict)) # parse the VCF line splitLine = vcfLine.split("\t") # the coordinate is the second element vcfChr = splitLine[0] vcfStopCoordinate = int(splitLine[1]) vcfIds = splitLine[2] vcfRef = splitLine[3] vcfAlts = splitLine[4] vcfScore = splitLine[5] vcfFilterSet = set(splitLine[6].split(";")) vcfInfoList = splitLine[7].split(";") vcfInfoDict = collections.defaultdict(list) for info in vcfInfoList: keyValueList = info.split("=") # some keys are just singular without a value (e.g. DB, etc.) if (len(keyValueList) == 1): vcfInfoDict[keyValueList[0]] = ["True"] else: # the value can be a comma separated list vcfInfoDict[keyValueList[0]] = keyValueList[1].split(",") vcfRestOfLine = "\t".join(splitLine[8:]) modTypes = vcfInfoDict["MT"] modTypeFilters = dict() atLeastOnePass = False for modType in modTypes: blatOverallReadDepth = 0 numValidReads = 0 prefix = "" if (modType == "GERM" and i_blatDnaNormalReads): prefix = "dnaNormal" elif (modType == "NOR_EDIT" and i_blatRnaNormalReads): prefix = "rnaNormal" elif (modType == "SOM" and i_blatDnaTumorReads): prefix = "dnaTumor" elif ((modType == "SOM" or modType == "TUM_EDIT") and i_blatRnaTumorReads): prefix = "rnaTumor" # get the expected prefix vcfKey = "_".join([prefix, vcfChr, str(vcfStopCoordinate)]) # for each read, investigate the blat # hits to see if this read is valid for (readId, blatHitList) in blatHitsDict.iteritems(): if (i_debug): logging.debug("num of blat hits for read %s=%s", readId, len(blatHitList)) # if the readId does not start with the vcfKey, # then something is wrong. the VCF and blat hits # need to be in sync... if (not readId.startswith(vcfKey)): logging.error("The blat query seems to be out of sync " + "with the blat hits.") logging.error("VCF Line=%s", vcfLine) logging.error("readId=%s, blatHitsDict=%s", readId, blatHitsDict[readId][1]) sys.exit(1) blatOverallReadDepth += 1 # find out if the read is valid or if it # maps to other places in the genome if (i_blatOutputFormat == "PSL"): # if we should process the transcripts if ((i_transcriptNameTag is not None) and (i_transcriptNameTag in vcfInfoDict)): (isValidRead, validRead) = is_valid_read_psl_format( blatHitList, vcfInfoDict[i_transcriptNameTag], vcfInfoDict[i_transcriptCoordinateTag], i_rnaIncludeSecondaryAlignments, i_debug) else: (isValidRead, validRead) = is_valid_read_psl_format( blatHitList, [vcfChr], [vcfStopCoordinate], i_rnaIncludeSecondaryAlignments, i_debug) elif (i_blatOutputFormat == "BLAST"): # if we should process the transcripts if ((i_transcriptNameTag is not None) and (i_transcriptNameTag in vcfInfoDict)): (isValidRead, validRead) = is_valid_read_blast_format( blatHitList, vcfInfoDict[i_transcriptNameTag], vcfInfoDict[i_transcriptCoordinateTag], i_rnaIncludeSecondaryAlignments, i_minOrderMagnitude, i_debug) else: (isValidRead, validRead) = is_valid_read_blast_format( blatHitList, [vcfChr], [vcfStopCoordinate], i_rnaIncludeSecondaryAlignments, i_minOrderMagnitude, i_debug) # if we have only one valid blat hit, then the read doesn't # map to other places in the genome very well, so let's use it if (isValidRead): numValidReads += 1 if (i_debug): logging.debug("ValidRead: %s", validRead) elif (i_debug): logging.debug("not a valid read") if (blatOverallReadDepth > 0): tmpAltPct = numValidReads / float(blatOverallReadDepth) altPercent = round(tmpAltPct, 2) else: altPercent = 0.0 if (numValidReads < i_minReadDepth or altPercent < i_minReadPercent): modTypeFilters[modType] = "blat" else: modTypeFilters[modType] = "PASS" atLeastOnePass = True if (i_debug): logging.debug( "blatOverallReadDepth=%s, numValidReads=%s, " + "altPercent=%s", str(blatOverallReadDepth), str(numValidReads), str(altPercent)) logging.debug("modType=%s, passed? %s", modType, modTypeFilters[modType]) logging.debug("blatFilter originalDepth=%s, validBlatDepth=%s", str(blatOverallReadDepth), str(numValidReads)) # make a copy of the list to manipulate modTypesTmpList = list(modTypes) modChanges = vcfInfoDict["MC"] # if at least one passed, then remove the ones that didn't for (modType, modChange) in izip(modTypes, modChanges): # if at least one passed, then remove the ones that didn't if (atLeastOnePass): if (modTypeFilters[modType] == "blat"): modTypesTmpList.remove(modType) modChanges.remove(modChange) # set the modTypes and modChanges vcfInfoDict["MT"] = modTypesTmpList vcfInfoDict["MC"] = modChanges # if at least one passed, then set pass if (atLeastOnePass): vcfFilterSet = ["PASS"] else: # if the user wants to keep the previous filters if (i_keepPreviousFiltersFlag): # if the call previous passed, then just set blat if (len(vcfFilterSet) == 1 and "PASS" in vcfFilterSet): vcfFilterSet = ["blat"] # otherwise, add it to the previous filters else: vcfFilterSet.add("blat") # otherwise, just set the blat filter else: vcfFilterSet = ["blat"] # update the mod filters modTypes = vcfInfoDict["MT"] modChanges = vcfInfoDict["MC"] origins = vcfInfoDict["ORIGIN"] if vcfInfoDict["MF"] is None: modFilters = [] else: modFilters = vcfInfoDict["MF"] if vcfInfoDict["MFT"] is None: modFilterTypes = [] else: modFilterTypes = vcfInfoDict["MFT"] for origin in origins: for (modType, modChange) in izip(modTypes, modChanges): modFilterTypes.append("_".join( [origin, modType, modChange])) modFilters.append("_".join(vcfFilterSet)) vcfInfoDict["MF"] = modFilters vcfInfoDict["MFT"] = modFilterTypes output = [ vcfChr, str(vcfStopCoordinate), vcfIds, vcfRef, vcfAlts, vcfScore, ";".join(vcfFilterSet) ] # add the modified info dict infoField = "" for key in sorted(vcfInfoDict.iterkeys()): if (len(vcfInfoDict[key]) == 0): continue elif ("True" in vcfInfoDict[key]): infoField += key + ";" else: infoField += key + "=" + ",".join(vcfInfoDict[key]) + ";" output.append(infoField.rstrip(";")) output.append(vcfRestOfLine) i_outputFileHandler.write("\t".join(output) + "\n") stopTime = time.time() logging.info( "filterByBlat.py for Id %s: Total time=%s hrs, %s mins, " + "%s secs", i_id, ((stopTime - startTime) / (3600)), ((stopTime - startTime) / 60), (stopTime - startTime)) # close the files if (i_outputFilename is not sys.stdout): i_outputFileHandler.close() return
def main(): # create the usage statement usage = "usage: python %prog vcfFile rnaGeneFile rnaGeneFamilyFile [Options]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option( "-o", "--outputFilename", dest="outputFilename", metavar="OUTPUT_FILE", help="the name of the output file, STDOUT by default") i_cmdLineParser.add_option( "-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help= "the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default" ) i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") i_cmdLineParser.add_option( "-c", "--allVCFCalls", action="store_false", default=True, dest="passedVCFCallsOnly", help= "by default only the VCF calls that have passed all filters thus far are processed, include this argument if all of the VCF calls should be processed" ) # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(3, 14, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_vcfFilename = str(i_cmdLineArgs[0]) i_rnaGeneFilename = str(i_cmdLineArgs[1]) i_rnaGeneFamilyFilename = str(i_cmdLineArgs[2]) # get the optional params with default values i_logLevel = i_cmdLineOptions.logLevel i_passedVCFCallsOnlyFlag = i_cmdLineOptions.passedVCFCallsOnly # try to get any optional parameters with no defaults i_outputFilename = None i_logFilename = None if (i_cmdLineOptions.outputFilename != None): i_outputFilename = str(i_cmdLineOptions.outputFilename) if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError( "Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("vcfFilename=%s", i_vcfFilename) logging.debug("rnaGeneFilename=%s", i_rnaGeneFilename) logging.debug("rnaGeneFamilyFilename=%s", i_rnaGeneFamilyFilename) logging.debug("outputFilename=%s", i_outputFilename) logging.debug("logFilename=%s", i_logFilename) logging.debug("passedOnly?=%s", i_passedVCFCallsOnlyFlag) # check for any errors i_writeFilenameList = [] if (i_outputFilename != None): i_writeFilenameList = [i_outputFilename] if (i_logFilename != None): i_writeFilenameList = [i_logFilename] i_readFilenameList = [ i_vcfFilename, i_rnaGeneFilename, i_rnaGeneFamilyFilename ] if (not radiaUtil.check_for_argv_errors(None, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # open the output stream i_outputFileHandler = None if (i_outputFilename != None): i_outputFileHandler = radiaUtil.get_write_fileHandler(i_outputFilename) # get the RNA gene blacklists (i_rnaGeneList, i_rnaGeneFamilyList) = get_rna_genes(i_rnaGeneFilename, i_rnaGeneFamilyFilename, i_debug) hasAddedHeader = False i_vcfFileHandler = radiaUtil.get_read_fileHandler(i_vcfFilename) vcfHeader = "##FILTER=<ID=rgene,Description=\"This gene is on the RNA gene blacklist\">\n" vcfHeader += "##FILTER=<ID=rgfam,Description=\"This gene family is on the RNA gene family blacklist\">\n" for line in i_vcfFileHandler: # strip the carriage return and newline characters line = line.rstrip("\r\n") if (i_debug): logging.debug("vcfLine: %s", line) # if it is an empty line, then just continue if (line.isspace()): continue # if we find the FILTER section, then add the filters from here elif ((not hasAddedHeader) and (line.startswith("##FILTER") or line.startswith("##INFO"))): hasAddedHeader = True if (i_outputFileHandler != None): i_outputFileHandler.write(vcfHeader) i_outputFileHandler.write(line + "\n") else: print >> sys.stdout, vcfHeader print >> sys.stdout, line # these lines are from previous scripts in the pipeline, so output them elif (line.startswith("#")): if (i_outputFileHandler != None): i_outputFileHandler.write(line + "\n") else: print >> sys.stdout, line # if we are only suppose to process the passed calls # and this call has not passed, then skip it elif (i_passedVCFCallsOnlyFlag and "PASS" not in line): if (i_outputFileHandler != None): i_outputFileHandler.write(line + "\n") else: print >> sys.stdout, line # now we are to the data else: # split the line on the tab splitLine = line.split("\t") filterSet = set(splitLine[6].split(";")) # if there are no filters so far, then clear the list if (len(filterSet) == 1 and "PASS" in filterSet): filterSet = set() # parse the info column and create a dict infoList = splitLine[7].split(";") infoDict = collections.defaultdict(list) for info in infoList: keyValueList = info.split("=") # some keys are just singular without a value (e.g. DB, SOMATIC, etc.) if (len(keyValueList) == 1): infoDict[keyValueList[0]] = ["True"] else: # the value can be a comma separated list infoDict[keyValueList[0]] = keyValueList[1].split(",") effectList = infoDict["EFF"] effectRegEx = re.compile("(\\w).*\\({1}") ignoreEffectsList = ["UPSTREAM", "DOWNSTREAM"] isRnaBlacklistGene = False isRnaBlacklistGeneFamily = False for rawEffect in effectList: rawEffect = rawEffect.rstrip(")") iterator = effectRegEx.finditer(rawEffect) # for each match object in the iterator for match in iterator: effect = match.group() rawEffect = rawEffect.replace(effect, "") effect = effect.rstrip("(") if (effect in ignoreEffectsList): continue effectParts = rawEffect.split("|") #effectImpact = effectParts[0] #functionalClass = effectParts[1] #codonChange = effectParts[2] #aaChange = effectParts[3] #aaLength = effectParts[4] geneName = effectParts[5] transcriptBiotype = effectParts[6] #geneCoding = effectParts[7] #ensembleId = effectParts[8] #exonNumber = effectParts[9] #genotypeNumber = effectParts[10] # the RNA gene list can have "RP11" and that # should filter out any gene with RP11 in it for rnaGene in i_rnaGeneList: if (rnaGene in geneName): isRnaBlacklistGene = True break if (transcriptBiotype in i_rnaGeneFamilyList): isRnaBlacklistGeneFamily = True output = ["\t".join(splitLine[0:6])] # if the filter should be applied if (isRnaBlacklistGene): filterSet.add("rgene") # if the filter should be applied if (isRnaBlacklistGeneFamily): filterSet.add("rgfam") # if there are no filters so far, then this call passes if (len(filterSet) == 0): filterSet.add("PASS") output.append(";".join(filterSet)) output.append("\t".join(splitLine[7:])) if (i_outputFilename != None): i_outputFileHandler.write("\t".join(output) + "\n") else: print >> sys.stdout, "\t".join(output) # close the files i_vcfFileHandler.close() if (i_outputFilename != None): i_outputFileHandler.close() return
def main(): # command for running this on a small test case: #python mergeChroms.py TCGA-BH-A18P-11A-43D-A12B-09_TCGA-BH-A18P-01A-11D-A12B-09 ../data/test/ ../data/test/ --log=DEBUG startTime = time.time() # create the usage statement usage = "usage: python %prog id inputDir outputDir [Options]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option( "-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help= "the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default" ) i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") i_cmdLineParser.add_option( "", "--gzip", action="store_true", default=False, dest="gzip", help= "include this argument if the final VCF should be compressed with gzip" ) # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(3, 10, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = i_cmdLineArgs[0] i_inputDir = i_cmdLineArgs[1] i_outputDir = i_cmdLineArgs[2] # get the optional params with default values i_logLevel = i_cmdLineOptions.logLevel i_gzip = i_cmdLineOptions.gzip i_logFilename = None if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError( "Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("id=%s", i_id) logging.debug("inputDir=%s", i_inputDir) logging.debug("outputDir=%s", i_outputDir) logging.debug("logLevel=%s", i_logLevel) logging.debug("logFile=%s", i_logFilename) logging.debug("gzip=%s", i_gzip) # check for any errors i_readFilenameList = None if (i_logFilename != None): i_writeFilenameList = [i_logFilename] else: i_writeFilenameList = None i_dirList = [i_inputDir, i_outputDir] if (not radiaUtil.check_for_argv_errors(i_dirList, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # get the VCF generator (headerDict, coordinateDict) = get_vcf_data(i_id, i_inputDir, i_debug) if (i_gzip): i_outputFilename = os.path.join(i_outputDir, i_id + ".vcf.gz") else: i_outputFilename = os.path.join(i_outputDir, i_id + ".vcf") outputFileHandler = get_write_fileHandler(i_outputFilename) # if we have header info to output if (len(headerDict["metadata"]) > 0): # output the header information outputFileHandler.write("\n".join(headerDict["metadata"]) + "\n") outputFileHandler.write("\n".join(headerDict["filter"]) + "\n") outputFileHandler.write("\n".join(headerDict["info"]) + "\n") outputFileHandler.write("\n".join(headerDict["format"]) + "\n") outputFileHandler.write("".join(headerDict["chrom"]) + "\n") # first output the numerical chroms in order numericChromKeys = coordinateDict["numbers"].keys() numericChromKeys.sort(key=int) for chrom in numericChromKeys: outputFileHandler.write("\n".join(coordinateDict["numbers"][chrom]) + "\n") # then output the alphabetical chroms in order letterChromKeys = coordinateDict["letters"].keys() letterChromKeys.sort(key=str) for chrom in letterChromKeys: outputFileHandler.write("\n".join(coordinateDict["letters"][chrom]) + "\n") stopTime = time.time() logging.info("Total time for Id %s: Total time=%s hrs, %s mins, %s secs", i_id, ((stopTime - startTime) / (3600)), ((stopTime - startTime) / 60), (stopTime - startTime)) # close the files outputFileHandler.close() return
def main(): startTime = time.time() # create the usage statement usage = "usage: python %prog passingFile originalFile outputFile [Options]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option("-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default") i_cmdLineParser.add_option("-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(3,10,1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_passingFilename = i_cmdLineArgs[0] i_originalFilename = i_cmdLineArgs[1] i_outputFilename = i_cmdLineArgs[2] # get the optional params with default values i_logLevel = i_cmdLineOptions.logLevel i_logFilename = None if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("passingFile=%s", i_passingFilename) logging.debug("originalFile=%s", i_originalFilename) logging.debug("outputFilename=%s", i_outputFilename) # check for any errors i_readFilenameList = [i_passingFilename, i_originalFilename] i_writeFilenameList = [i_outputFilename] i_dirList = None if (not radiaUtil.check_for_argv_errors(i_dirList, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # get the VCF generator (passHeaderList, chromLine, passInfoList, passFilterList, passCoordinateDict) = get_vcf_data(i_passingFilename, i_debug) (orgHeaderList, chromLine, orgInfoList, orgFilterList, orgCoordinateDict) = get_vcf_data(i_originalFilename, i_debug) outputFileHandler = get_write_fileHandler(i_outputFilename) for headerLine in orgHeaderList: outputFileHandler.write(headerLine + "\n") for headerLine in orgInfoList: outputFileHandler.write(headerLine + "\n") for headerLine in passInfoList: if (headerLine not in orgInfoList): outputFileHandler.write(headerLine + "\n") for headerLine in orgFilterList: outputFileHandler.write(headerLine + "\n") for headerLine in passFilterList: if (headerLine not in orgFilterList): outputFileHandler.write(headerLine + "\n") outputFileHandler.write(chromLine + "\n") numericKeys = orgCoordinateDict.keys() numericKeys.sort(key=int) for coordinate in numericKeys: if (coordinate in passCoordinateDict): line = passCoordinateDict[coordinate] else: line = orgCoordinateDict[coordinate] outputFileHandler.write(line) stopTime = time.time() logging.info("Total time=%s hrs, %s mins, %s secs", ((stopTime-startTime)/(3600)), ((stopTime-startTime)/60), (stopTime-startTime)) # close the files outputFileHandler.close() return
def main(): # command for running this on a small test case: # python filterByBlat.py TCGA-00-4454 7 ../data/test/TCGA-00-4454_EGFR.vcf # ../data/test/TCGA-00-4454_EGFR.fa ../data/test/TCGA-00-4454_EGFR.psl # --dnaNormalFilename=../data/test/TCGA-00-4454_EGFR.reads startTime = time.time() # create the usage statement usage = "usage: python %prog id chrom vcfFile blatOutputFile [Options]" i_cmdLineParser = OptionParser(usage=usage) # add the optional parameters i_cmdLineParser.add_option( "-c", "--allVCFCalls", action="store_false", default=True, dest="passedVCFCallsOnly", help="by default only the VCF calls that have passed all filters " + "thus far are processed, include this argument if all of the " + "VCF calls should be processed") i_cmdLineParser.add_option( "-k", "--keepPreviousFilters", action="store_true", default=False, dest="keepPreviousFilters", help="by default the previous filters are overwritten with the blat " + "filter, include this argument if the previous filters should " + "be kept") i_cmdLineParser.add_option( "-o", "--outputFilename", dest="outputFilename", metavar="OUTPUT_FILE", default=sys.stdout, help="the name of the output file, STDOUT by default") i_cmdLineParser.add_option( "-b", "--blatOutputFormat", dest="blatOutputFormat", metavar="OUTPUT_FORMAT", default="BLAST", help="the BLAT output format, BLAST by default") i_cmdLineParser.add_option( "-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), " + "%default by default") i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDERR by default") i_cmdLineParser.add_option( "", "--transcriptNameTag", dest="transcriptNameTag", help="the INFO key where the original transcript name can be found") i_cmdLineParser.add_option( "", "--transcriptCoordinateTag", dest="transcriptCoordinateTag", help="the INFO key where the original transcript" + "coordinate can be found") i_cmdLineParser.add_option( "", "--transcriptStrandTag", dest="transcriptStrandTag", help="the INFO key where the original transcript strand can be found") i_cmdLineParser.add_option( "", "--rnaIncludeSecondaryAlignments", action="store_true", default=False, dest="rnaIncludeSecondaryAlignments", help="if you align the RNA to transcript isoforms, then you may " + "want to include RNA secondary alignments in the pileup") i_cmdLineParser.add_option( "-n", "--blatDnaNormalReads", action="store_true", default=False, dest="blatDnaNormalReads", help="include this argument if the normal DNA reads " + "should be processed") i_cmdLineParser.add_option( "-x", "--blatRnaNormalReads", action="store_true", default=False, dest="blatRnaNormalReads", help="include this argument if the normal RNA reads " + "should be processed") i_cmdLineParser.add_option( "-t", "--blatDnaTumorReads", action="store_true", default=False, dest="blatDnaTumorReads", help="include this argument if the tumor DNA reads " + "should be processed") i_cmdLineParser.add_option( "-r", "--blatRnaTumorReads", action="store_true", default=False, dest="blatRnaTumorReads", help="include this argument if the tumor RNA reads " + "should be processed") i_cmdLineParser.add_option( "-d", "--minReadDepth", type="int", default=int(4), dest="minReadDepth", metavar="MIN_READ_DP", help="the minimum number of valid reads that are necessary, " + "%default by default") i_cmdLineParser.add_option( "-p", "--minReadPercent", type="float", default=float(0.10), dest="minReadPercent", metavar="MIN_READ_PCT", help="the minimum percentage of valid reads that are necessary, " + "%default by default") i_cmdLineParser.add_option( "-m", "--minOrderMagnitude", type="int", default=float(0), dest="minOrderMagnitude", metavar="MIN_ORDER_MAGNITUDE", help="the minimum order of magnitude difference between the blat " + "hit at the query position vs. the next best blat hit in order " + "for the read to be valid, %default by default") ''' i_cmdLineParser.add_option( "-e", "--minEValue", type="float", default=float(10e-6), dest="minEValue", metavar="MIN_EVALUE", help="the minimum e-value needed for a blat hit to be significant, " + "%default by default") i_cmdLineParser.add_option( "-u", "--maxIdentity", type="float", default=float(0.95), dest="maxIdentity", metavar="MAX_IDENTITY", help="the maximum match length adjusted identity for a blat hit to " + "be significant, %default by default") i_cmdLineParser.add_option( "-l", "--minIdentity", type="float", default=float(0.5), dest="minIdentity", metavar="MIN_IDENTITY", help="the minimum match length adjusted identity for a blat hit to " + "be significant, %default by default") ''' # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(5, 27, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (cmdLineOpts, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = str(i_cmdLineArgs[0]) i_vcfFilename = str(i_cmdLineArgs[1]) i_blatOutputFilename = str(i_cmdLineArgs[2]) # get the optional params with default values i_passedVCFCallsOnlyFlag = cmdLineOpts.passedVCFCallsOnly i_keepPreviousFiltersFlag = cmdLineOpts.keepPreviousFilters i_blatOutputFormat = cmdLineOpts.blatOutputFormat i_logLevel = cmdLineOpts.logLevel i_rnaIncludeSecondaryAlignments = cmdLineOpts.rnaIncludeSecondaryAlignments i_minReadDepth = cmdLineOpts.minReadDepth i_minReadPercent = cmdLineOpts.minReadPercent i_minOrderMagnitude = cmdLineOpts.minOrderMagnitude # i_minEValue = cmdLineOpts.minEValue # i_maxIdentity = cmdLineOpts.maxIdentity # i_minIdentity = cmdLineOpts.minIdentity i_blatDnaNormalReads = cmdLineOpts.blatDnaNormalReads i_blatDnaTumorReads = cmdLineOpts.blatDnaTumorReads i_blatRnaNormalReads = cmdLineOpts.blatRnaNormalReads i_blatRnaTumorReads = cmdLineOpts.blatRnaTumorReads # try to get any optional parameters with no defaults i_outputFilename = None i_logFilename = None i_transcriptNameTag = None i_transcriptCoordinateTag = None i_transcriptStrandTag = None if (cmdLineOpts.outputFilename is not None): i_outputFilename = cmdLineOpts.outputFilename if (cmdLineOpts.logFilename is not None): i_logFilename = cmdLineOpts.logFilename if (cmdLineOpts.transcriptNameTag is not None): i_transcriptNameTag = cmdLineOpts.transcriptNameTag if (cmdLineOpts.transcriptCoordinateTag is not None): i_transcriptCoordinateTag = cmdLineOpts.transcriptCoordinateTag if (cmdLineOpts.transcriptStrandTag is not None): i_transcriptStrandTag = cmdLineOpts.transcriptStrandTag # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the " + "following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename is not None): logging.basicConfig( level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig( level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("id=%s", i_id) logging.debug("vcfFilename=%s", i_vcfFilename) logging.debug("blatOutputFilename=%s", i_blatOutputFilename) logging.debug("passedCallsOnly? %s", i_passedVCFCallsOnlyFlag) logging.debug("keepPreviousFiltersFlag? %s", i_keepPreviousFiltersFlag) logging.debug("blatOutputFormat=%s", i_blatOutputFormat) logging.debug("transcriptNameTag %s", i_transcriptNameTag) logging.debug("transcriptCoordinateTag %s", i_transcriptCoordinateTag) logging.debug("transcriptStrandTag %s", i_transcriptStrandTag) logging.debug("rnaInclSecAlign=%s" % i_rnaIncludeSecondaryAlignments) logging.debug("blatDnaNormal? %s", i_blatDnaNormalReads) logging.debug("blatDnaTumor? %s", i_blatDnaTumorReads) logging.debug("blatRnaNormal? %s", i_blatRnaNormalReads) logging.debug("blatRnaTumor? %s", i_blatRnaTumorReads) logging.debug("minReadDepth=%s", i_minReadDepth) logging.debug("minReadPercent=%s", i_minReadPercent) logging.debug("minOrderMagnitude=%s", i_minOrderMagnitude) # check for any errors i_writeFilenameList = [] if (cmdLineOpts.outputFilename is not sys.stdout): i_writeFilenameList = [i_outputFilename] if (i_logFilename is not None): i_writeFilenameList = [i_logFilename] i_readFilenameList = [i_vcfFilename, i_blatOutputFilename] if (not radiaUtil.check_for_argv_errors(None, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # open the output stream if i_outputFilename is not sys.stdout: i_outputFileHandler = radiaUtil.get_write_fileHandler(i_outputFilename) else: i_outputFileHandler = i_outputFilename # get the VCF generator i_vcfGenerator = get_vcf_data(i_vcfFilename, i_passedVCFCallsOnlyFlag, i_debug) # get the blat hits generator i_blatGenerator = parse_blat_output(i_blatOutputFilename, i_blatOutputFormat, i_debug) for (vcfLine, blatHitsDict) in izip(i_vcfGenerator, i_blatGenerator): if (i_debug): logging.debug("VCF Line=%s", vcfLine) logging.debug("Len Blat Hits=%s", len(blatHitsDict)) # parse the VCF line splitLine = vcfLine.split("\t") # the coordinate is the second element vcfChr = splitLine[0] vcfStopCoordinate = int(splitLine[1]) vcfIds = splitLine[2] vcfRef = splitLine[3] vcfAlts = splitLine[4] vcfScore = splitLine[5] vcfFilterSet = set(splitLine[6].split(";")) vcfInfoList = splitLine[7].split(";") vcfInfoDict = collections.defaultdict(list) for info in vcfInfoList: keyValueList = info.split("=") # some keys are just singular without a value (e.g. DB, etc.) if (len(keyValueList) == 1): vcfInfoDict[keyValueList[0]] = ["True"] else: # the value can be a comma separated list vcfInfoDict[keyValueList[0]] = keyValueList[1].split(",") vcfRestOfLine = "\t".join(splitLine[8:]) modTypes = vcfInfoDict["MT"] modTypeFilters = dict() atLeastOnePass = False for modType in modTypes: blatOverallReadDepth = 0 numValidReads = 0 prefix = "" if (modType == "GERM" and i_blatDnaNormalReads): prefix = "dnaNormal" elif (modType == "NOR_EDIT" and i_blatRnaNormalReads): prefix = "rnaNormal" elif (modType == "SOM" and i_blatDnaTumorReads): prefix = "dnaTumor" elif ((modType == "SOM" or modType == "TUM_EDIT") and i_blatRnaTumorReads): prefix = "rnaTumor" # get the expected prefix vcfKey = "_".join([prefix, vcfChr, str(vcfStopCoordinate)]) # for each read, investigate the blat # hits to see if this read is valid for (readId, blatHitList) in blatHitsDict.iteritems(): if (i_debug): logging.debug("num of blat hits for read %s=%s", readId, len(blatHitList)) # if the readId does not start with the vcfKey, # then something is wrong. the VCF and blat hits # need to be in sync... if (not readId.startswith(vcfKey)): logging.error("The blat query seems to be out of sync " + "with the blat hits.") logging.error("VCF Line=%s", vcfLine) logging.error("readId=%s, blatHitsDict=%s", readId, blatHitsDict[readId][1]) sys.exit(1) blatOverallReadDepth += 1 # find out if the read is valid or if it # maps to other places in the genome if (i_blatOutputFormat == "PSL"): # if we should process the transcripts if ((i_transcriptNameTag is not None) and (i_transcriptNameTag in vcfInfoDict)): (isValidRead, validRead) = is_valid_read_psl_format( blatHitList, vcfInfoDict[i_transcriptNameTag], vcfInfoDict[i_transcriptCoordinateTag], i_rnaIncludeSecondaryAlignments, i_debug) else: (isValidRead, validRead) = is_valid_read_psl_format( blatHitList, [vcfChr], [vcfStopCoordinate], i_rnaIncludeSecondaryAlignments, i_debug) elif (i_blatOutputFormat == "BLAST"): # if we should process the transcripts if ((i_transcriptNameTag is not None) and (i_transcriptNameTag in vcfInfoDict)): (isValidRead, validRead) = is_valid_read_blast_format( blatHitList, vcfInfoDict[i_transcriptNameTag], vcfInfoDict[i_transcriptCoordinateTag], i_rnaIncludeSecondaryAlignments, i_minOrderMagnitude, i_debug) else: (isValidRead, validRead) = is_valid_read_blast_format( blatHitList, [vcfChr], [vcfStopCoordinate], i_rnaIncludeSecondaryAlignments, i_minOrderMagnitude, i_debug) # if we have only one valid blat hit, then the read doesn't # map to other places in the genome very well, so let's use it if (isValidRead): numValidReads += 1 if (i_debug): logging.debug("ValidRead: %s", validRead) elif (i_debug): logging.debug("not a valid read") if (blatOverallReadDepth > 0): tmpAltPct = numValidReads/float(blatOverallReadDepth) altPercent = round(tmpAltPct, 2) else: altPercent = 0.0 if (numValidReads < i_minReadDepth or altPercent < i_minReadPercent): modTypeFilters[modType] = "blat" else: modTypeFilters[modType] = "PASS" atLeastOnePass = True if (i_debug): logging.debug("blatOverallReadDepth=%s, numValidReads=%s, " + "altPercent=%s", str(blatOverallReadDepth), str(numValidReads), str(altPercent)) logging.debug("modType=%s, passed? %s", modType, modTypeFilters[modType]) logging.debug("blatFilter originalDepth=%s, validBlatDepth=%s", str(blatOverallReadDepth), str(numValidReads)) # make a copy of the list to manipulate modTypesTmpList = list(modTypes) modChanges = vcfInfoDict["MC"] # if at least one passed, then remove the ones that didn't for (modType, modChange) in izip(modTypes, modChanges): # if at least one passed, then remove the ones that didn't if (atLeastOnePass): if (modTypeFilters[modType] == "blat"): modTypesTmpList.remove(modType) modChanges.remove(modChange) # set the modTypes and modChanges vcfInfoDict["MT"] = modTypesTmpList vcfInfoDict["MC"] = modChanges # if at least one passed, then set pass if (atLeastOnePass): vcfFilterSet = ["PASS"] else: # if the user wants to keep the previous filters if (i_keepPreviousFiltersFlag): # if the call previous passed, then just set blat if (len(vcfFilterSet) == 1 and "PASS" in vcfFilterSet): vcfFilterSet = ["blat"] # otherwise, add it to the previous filters else: vcfFilterSet.add("blat") # otherwise, just set the blat filter else: vcfFilterSet = ["blat"] # update the mod filters modTypes = vcfInfoDict["MT"] modChanges = vcfInfoDict["MC"] origins = vcfInfoDict["ORIGIN"] if vcfInfoDict["MF"] is None: modFilters = [] else: modFilters = vcfInfoDict["MF"] if vcfInfoDict["MFT"] is None: modFilterTypes = [] else: modFilterTypes = vcfInfoDict["MFT"] for origin in origins: for (modType, modChange) in izip(modTypes, modChanges): modFilterTypes.append("_".join([origin, modType, modChange])) modFilters.append("_".join(vcfFilterSet)) vcfInfoDict["MF"] = modFilters vcfInfoDict["MFT"] = modFilterTypes output = [vcfChr, str(vcfStopCoordinate), vcfIds, vcfRef, vcfAlts, vcfScore, ";".join(vcfFilterSet)] # add the modified info dict infoField = "" for key in sorted(vcfInfoDict.iterkeys()): if (len(vcfInfoDict[key]) == 0): continue elif ("True" in vcfInfoDict[key]): infoField += key + ";" else: infoField += key + "=" + ",".join(vcfInfoDict[key]) + ";" output.append(infoField.rstrip(";")) output.append(vcfRestOfLine) i_outputFileHandler.write("\t".join(output) + "\n") stopTime = time.time() logging.info("filterByBlat.py for Id %s: Total time=%s hrs, %s mins, " + "%s secs", i_id, ((stopTime-startTime)/(3600)), ((stopTime-startTime)/60), (stopTime-startTime)) # close the files if (i_outputFilename is not sys.stdout): i_outputFileHandler.close() return
def main(): # command for running this on a small test case: #python createBlatFile.py TCGA-00-4454 7 ../data/test/TCGA-00-4454_EGFR.vcf ../data/test/tmp/ --dnaNormalFilename=../data/test/TCGA-00-4454_EGFR.reads startTime = time.time() # create the usage statement usage = "usage: python %prog id vcfFile headerFile [Options]" i_cmdLineParser = OptionParser(usage=usage) # add the optional parameters i_cmdLineParser.add_option("-c", "--allVCFCalls", action="store_false", default=True, dest="passedVCFCallsOnly", help="by default only the VCF calls that have passed all filters thus far are processed, include this argument if all of the VCF calls should be processed") i_cmdLineParser.add_option("-o", "--outputFilename", dest="outputFilename", metavar="OUTPUT_FILE", help="the name of the output file, STDOUT by default") i_cmdLineParser.add_option("-b", "--allReadBases", action="store_false", default=True, dest="altBasesOnly", help="by default only the reads with the alternate base are processed, include this argument if all of the reads should be processed") i_cmdLineParser.add_option("-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default") i_cmdLineParser.add_option("-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") i_cmdLineParser.add_option("-n", "--blatDnaNormalReads", action="store_true", default=False, dest="blatDnaNormalReads", help="include this argument if the normal DNA reads should be processed") i_cmdLineParser.add_option("-x", "--blatRnaNormalReads", action="store_true", default=False, dest="blatRnaNormalReads", help="include this argument if the normal RNA reads should be processed") i_cmdLineParser.add_option("-t", "--blatDnaTumorReads", action="store_true", default=False, dest="blatDnaTumorReads", help="include this argument if the tumor DNA reads should be processed") i_cmdLineParser.add_option("-r", "--blatRnaTumorReads", action="store_true", default=False, dest="blatRnaTumorReads", help="include this argument if the tumor RNA reads should be processed") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(3,22,1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = i_cmdLineArgs[0] i_vcfFilename = i_cmdLineArgs[1] i_headerFilename = i_cmdLineArgs[2] # get the optional params with default values i_passedVCFCallsOnlyFlag = i_cmdLineOptions.passedVCFCallsOnly i_altBasesOnlyFlag = i_cmdLineOptions.altBasesOnly i_logLevel = i_cmdLineOptions.logLevel i_blatDnaNormalReads = i_cmdLineOptions.blatDnaNormalReads i_blatDnaTumorReads = i_cmdLineOptions.blatDnaTumorReads i_blatRnaNormalReads = i_cmdLineOptions.blatRnaNormalReads i_blatRnaTumorReads = i_cmdLineOptions.blatRnaTumorReads # try to get any optional parameters with no defaults i_readFilenameList = [i_vcfFilename, i_headerFilename] i_writeFilenameList = [] i_logFilename = None i_outputFilename = None if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) i_writeFilenameList += [i_logFilename] if (i_cmdLineOptions.outputFilename != None): i_outputFilename = str(i_cmdLineOptions.outputFilename) i_writeFilenameList += [i_outputFilename] # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel < logging.WARNING) # output some debug info if (i_debug): logging.debug("id=%s", i_id) logging.debug("vcfFilename=%s", i_vcfFilename) logging.debug("headerFilename=%s", i_headerFilename) logging.debug("outputFilename=%s", i_outputFilename) logging.debug("passedCallsOnly? %s", i_passedVCFCallsOnlyFlag) logging.debug("altBasesOnlyFlag? %s", i_altBasesOnlyFlag) logging.debug("blatDnaNormal? %s", i_blatDnaNormalReads) logging.debug("blatDnaTumor? %s", i_blatDnaTumorReads) logging.debug("blatRnaNormal? %s", i_blatRnaNormalReads) logging.debug("blatRnaTumor? %s", i_blatRnaTumorReads) if (not radiaUtil.check_for_argv_errors(None, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # open the output stream i_outputFileHandler = None if (i_outputFilename != None): i_outputFileHandler = get_write_fileHandler(i_outputFilename) # get the VCF generator i_vcfGenerator = get_vcf_data(i_vcfFilename, i_headerFilename, i_passedVCFCallsOnlyFlag, i_altBasesOnlyFlag, i_debug) # for each VCF call that should be investigated for (vcfChr, vcfStopCoordinate, vcfId, vcfRef, vcfAlt, vcfScore, vcfFilterSet, vcfInfoDict, restOfLine, vcfParamsDict) in i_vcfGenerator: if (i_debug): logging.debug("VCF Data: %s %s %s %s %s %s %s %s %s", vcfChr, str(vcfStopCoordinate), vcfId, vcfRef, vcfAlt, vcfScore, str(vcfFilterSet), str(vcfInfoDict), restOfLine) modTypes = vcfInfoDict["MT"] for modType in modTypes: # get the reads contributing to a call and put them in a blat query file if (i_blatDnaNormalReads): write_to_blat_file(i_outputFileHandler, vcfChr, vcfStopCoordinate, vcfParamsDict, vcfInfoDict, "dnaNormal", i_debug) if (modType == "NOR_EDIT" and i_blatRnaNormalReads): write_to_blat_file(i_outputFileHandler, vcfChr, vcfStopCoordinate, vcfParamsDict, vcfInfoDict, "rnaNormal", i_debug) if (i_blatDnaTumorReads): write_to_blat_file(i_outputFileHandler, vcfChr, vcfStopCoordinate, vcfParamsDict, vcfInfoDict, "dnaTumor", i_debug) if ((modType == "SOM" or modType == "TUM_EDIT") and i_blatRnaTumorReads): write_to_blat_file(i_outputFileHandler, vcfChr, vcfStopCoordinate, vcfParamsDict, vcfInfoDict, "rnaTumor", i_debug) stopTime = time.time() logging.info("Id %s: Total time=%s hrs, %s mins, %s secs", i_id, ((stopTime-startTime)/(3600)), ((stopTime-startTime)/60), (stopTime-startTime)) # close the files if (i_outputFilename != None): i_outputFileHandler.close() return
def main(): # command for running this on a small test case: # python mergeChroms.py TCGA-BH-A18P # ../data/test/ ../data/test/ --log=DEBUG startTime = time.time() # create the usage statement usage = "usage: python %prog id inputDir outputDir [Options]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option( "-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), " + "%default by default") i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDERR by default") i_cmdLineParser.add_option( "", "--gzip", dest="gzip", action="store_true", default=False, help="include this argument if the final VCF should be " + "compressed with gzip") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(3, 10, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = i_cmdLineArgs[0] i_inputDir = i_cmdLineArgs[1] i_outputDir = i_cmdLineArgs[2] # get the optional params with default values i_logLevel = i_cmdLineOptions.logLevel i_gzip = i_cmdLineOptions.gzip i_logFilename = None if (i_cmdLineOptions.logFilename is not None): i_logFilename = str(i_cmdLineOptions.logFilename) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the " + "following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename is not None): logging.basicConfig( level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig( level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("id=%s", i_id) logging.debug("inputDir=%s", i_inputDir) logging.debug("outputDir=%s", i_outputDir) logging.debug("logLevel=%s", i_logLevel) logging.debug("logFile=%s", i_logFilename) logging.debug("gzip=%s", i_gzip) # check for any errors i_readFilenameList = None if (i_logFilename is not None): i_writeFilenameList = [i_logFilename] else: i_writeFilenameList = None i_dirList = [i_inputDir, i_outputDir] if (not radiaUtil.check_for_argv_errors(i_dirList, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # get the VCF generator (headerDict, coordDict) = get_vcf_data(i_id, i_inputDir, i_debug) if (i_gzip): i_outputFilename = os.path.join(i_outputDir, i_id + ".vcf.gz") else: i_outputFilename = os.path.join(i_outputDir, i_id + ".vcf") outputFileHandler = radiaUtil.get_write_fileHandler(i_outputFilename) # if we have header info to output if (len(headerDict["metadata"]) > 0): # output the header information outputFileHandler.write("\n".join(headerDict["metadata"]) + "\n") outputFileHandler.write("\n".join(headerDict["filter"]) + "\n") outputFileHandler.write("\n".join(headerDict["info"]) + "\n") outputFileHandler.write("\n".join(headerDict["format"]) + "\n") outputFileHandler.write("".join(headerDict["chrom"]) + "\n") # first output the numerical chroms in order numericChromKeys = coordDict["numbers"].keys() numericChromKeys.sort(key=int) for chrom in numericChromKeys: outputFileHandler.write("\n".join(coordDict["numbers"][chrom]) + "\n") # then output the alphabetical chroms in order letterChromKeys = coordDict["letters"].keys() letterChromKeys.sort(key=str) for chrom in letterChromKeys: outputFileHandler.write("\n".join(coordDict["letters"][chrom]) + "\n") stopTime = time.time() logging.info("Total time for Id %s: Total time=%s hrs, %s mins, %s secs", i_id, ((stopTime-startTime)/(3600)), ((stopTime-startTime)/60), (stopTime-startTime)) # close the files outputFileHandler.close() return
def main(): #python radiaCompare.py TCGA-AB-2995 12 ../data/test/TCGA-AB-2995.vcf ../data/test/TCGA-AB-2995.vcf -c "SOM=Somatic" -s ../stats/radia/cmpRadBB.tab --log=DEBUG # create the usage statement usage = "usage: python %prog id chrom radFile compareFile [Options]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option("-c", "--compareList", dest="compareList", metavar="COMPARE_LIST", help="a comma separated list of key/values comparisons where the key is in RADIA and the value is in the compare file") i_cmdLineParser.add_option("-s", "--statsFilename", dest="statsFilename", metavar="STATS_FILE", help="the name of the stats file, sys.stdout by default") i_cmdLineParser.add_option("-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default") i_cmdLineParser.add_option("-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDERR by default") i_cmdLineParser.add_option("-o", "--overlapFilename", dest="overlapFilename", metavar="OVERLAP_FILE", help="the name of the overlap file") i_cmdLineParser.add_option("-n", "--nonOverlapFilename", dest="nonOverlapFilename", metavar="NON_OVERLAP_FILE", help="the name of the non-overlap file") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(4,17,1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = str(i_cmdLineArgs[0]) i_chr = str(i_cmdLineArgs[1]) i_radiaFilename = str(i_cmdLineArgs[2]) i_compareFilename = str(i_cmdLineArgs[3]) # get the optional params with default values i_logLevel = i_cmdLineOptions.logLevel # try to get any optional parameters with no defaults # check for any errors writeFilenameList = [] readFilenameList = [i_radiaFilename, i_compareFilename] i_statsFilename = None i_logFilename = None i_compareString = None i_overlapFilename = None i_nonOverlapFilename = None i_compareDict = collections.defaultdict(list) if (i_cmdLineOptions.overlapFilename != None): i_overlapFilename = str(i_cmdLineOptions.overlapFilename) writeFilenameList += [i_overlapFilename] if (i_cmdLineOptions.nonOverlapFilename != None): i_nonOverlapFilename = str(i_cmdLineOptions.nonOverlapFilename) writeFilenameList += [i_nonOverlapFilename] if (i_cmdLineOptions.statsFilename != None): i_statsFilename = str(i_cmdLineOptions.statsFilename) writeFilenameList += [i_statsFilename] if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) writeFilenameList += [i_logFilename] if (i_cmdLineOptions.compareList != None): i_compareString = str(i_cmdLineOptions.compareList) i_compareList = i_compareString.split(",") for keyValue in i_compareList: (key, value) = keyValue.split("=") i_compareDict[key] = value # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel < logging.WARNING) # do some debugging if (i_debug): logging.debug("id=%s", i_id) logging.debug("chr=%s", i_chr) logging.debug("radiaFile=%s", i_radiaFilename) logging.debug("overlapFilename=%s" % i_overlapFilename) logging.debug("nonOverlapFilename=%s" % i_nonOverlapFilename) logging.debug("compareFile=%s", i_compareFilename) logging.debug("statsFile=%s", i_statsFilename) logging.debug("logLevel=%s", i_logLevel) logging.debug("logFile=%s", i_logFilename) logging.debug("compareDict=%s", i_compareDict) if (not radiaUtil.check_for_argv_errors(None, readFilenameList, writeFilenameList)): sys.exit(1) compare_events(i_id, i_chr, i_radiaFilename, i_compareFilename, i_statsFilename, i_overlapFilename, i_nonOverlapFilename, i_compareDict, i_debug) return
def main(): # python mergeRnaAndDnaFiles.py TCGA-AB-2995 5 # ../data/test/TCGA-AB-2995_dnaFile.vcf # ../data/test/TCGA-AB-2995_rnaFile.vcf # ../data/test/TCGA-AB-2995_rnaFile.vcf # ../data/test/ startTime = time.time() # create the usage statement usage = ("usage: python %prog id chrom dnaFile rnaFile rnaOverlapsFile " + "rnaNonOverlapsFile outputFile [Options]") i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option( "-l", "--log", default="WARNING", dest="logLevel", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), " + "%default by default") i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDERR by default") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(6, 15, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = i_cmdLineArgs[0] i_chrom = i_cmdLineArgs[1] i_dnaFilename = i_cmdLineArgs[2] i_rnaFilename = i_cmdLineArgs[3] i_overlapsFilename = i_cmdLineArgs[4] i_nonOverlapsFilename = i_cmdLineArgs[5] i_outputFilename = i_cmdLineArgs[6] # get the optional params with default values i_logLevel = i_cmdLineOptions.logLevel i_logFilename = None if (i_cmdLineOptions.logFilename is not None): i_logFilename = str(i_cmdLineOptions.logFilename) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the " + "following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename is not None): logging.basicConfig( level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig( level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("id=%s", i_id) logging.debug("chrom=%s", i_chrom) logging.debug("dnaFilename=%s", i_dnaFilename) logging.debug("rnaFilename=%s", i_rnaFilename) logging.debug("overlapsFilename=%s", i_overlapsFilename) logging.debug("nonOverlapsFilename=%s", i_nonOverlapsFilename) logging.debug("outputFilename=%s", i_outputFilename) # check for any errors i_readFilenameList = [i_dnaFilename, i_rnaFilename, i_overlapsFilename] i_writeFilenameList = [i_outputFilename] i_dirList = None if (not radiaUtil.check_for_argv_errors(i_dirList, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # get the VCF generator (headerList, coordinateDict) = merge_vcf_data(i_dnaFilename, i_rnaFilename, i_overlapsFilename, i_nonOverlapsFilename, i_debug) outputFileHandler = radiaUtil.get_write_fileHandler(i_outputFilename) for headerLine in headerList: outputFileHandler.write(headerLine) numericKeys = coordinateDict.keys() numericKeys.sort(key=int) for coordinate in numericKeys: line = coordinateDict[coordinate] line = line.rstrip("\r\n") # split the line on the tab splitLine = line.split("\t") # set the SST field in the INFO splitLine[7] = set_sst_field(splitLine[7]) outputFileHandler.write("\t".join(splitLine) + "\n") stopTime = time.time() logging.info("Total time for Id %s: Total time=%s hrs, %s mins, %s secs", i_id, ((stopTime-startTime)/(3600)), ((stopTime-startTime)/60), (stopTime-startTime)) # close the files outputFileHandler.close() return
def main(): # create the usage statement usage = "usage: python %prog id chrom vcfFile blatInputFile [Options]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option("-o", "--outputFilename", dest="outputFilename", metavar="OUTPUT_FILE", help="the name of the output file, STDOUT by default") i_cmdLineParser.add_option("-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help="the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default") i_cmdLineParser.add_option("-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") i_cmdLineParser.add_option("-d", "--readDepthCutoff", type="int", default=int(4), dest="readDepthCutoff", metavar="READ_DP_CUTOFF", help="the minimum number of reads that are necessary before applying this filter, %default by default") i_cmdLineParser.add_option("-p", "--readPercentCutoff", type="float", default=float(0.95), dest="readPercentCutoff", metavar="READ_PERCENT_CUTOFF", help="the maximum percentage of reads with the alternative allele at the beginning or end of the reads, %default by default") # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(4,17,1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = str(i_cmdLineArgs[0]) i_vcfFilename = str(i_cmdLineArgs[1]) i_blatInputFilename = str(i_cmdLineArgs[2]) # get the optional params with default values i_logLevel = i_cmdLineOptions.logLevel i_readDepthCutoff = i_cmdLineOptions.readDepthCutoff i_readPercentCutoff = i_cmdLineOptions.readPercentCutoff # try to get any optional parameters with no defaults i_outputFilename = None i_logFilename = None if (i_cmdLineOptions.outputFilename != None): i_outputFilename = str(i_cmdLineOptions.outputFilename) if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError("Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel < logging.WARNING) # output some debug info if (i_debug): logging.debug("id=%s", i_id) logging.debug("vcfFilename=%s", i_vcfFilename) logging.debug("blatInputFilename=%s", i_blatInputFilename) logging.debug("outputFilename=%s", i_outputFilename) logging.debug("logFilename=%s", i_logFilename) logging.debug("readDepthCutoff=%s", i_readDepthCutoff) logging.debug("readPerentCutoff=%s", i_readPercentCutoff) # check for any errors i_writeFilenameList = [] if (i_outputFilename != None): i_writeFilenameList = [i_outputFilename] if (i_logFilename != None): i_writeFilenameList = [i_logFilename] i_readFilenameList = [i_vcfFilename, i_blatInputFilename] if (not radiaUtil.check_for_argv_errors(None, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # open the output stream i_outputFileHandler = None if (i_outputFilename != None): i_outputFileHandler = get_write_fileHandler(i_outputFilename) # get the BLAT input i_blatCoordinateDict = parse_blat_input(i_blatInputFilename, i_debug) # get the VCF generator i_vcfGenerator = get_vcf_data(i_vcfFilename, i_debug) for (vcfChr, vcfStopCoordinate, vcfId, vcfRef, vcfAlt, vcfScore, vcfFilterSet, vcfInfoDict, restOfLine) in i_vcfGenerator: if (i_debug): logging.debug("VCF Data: %s %s %s %s %s %s %s %s %s", vcfChr, str(vcfStopCoordinate), vcfId, vcfRef, vcfAlt, vcfScore, str(vcfFilterSet), str(vcfInfoDict), restOfLine) modTypes = vcfInfoDict["MT"] modTypeFilters = dict() atLeastOnePass = False blatHitsList = list() # get the coordinate for this position coordinate = vcfChr + "_" + str(vcfStopCoordinate) for modType in modTypes: if (modType == "NOR_EDIT"): if (coordinate in i_blatCoordinateDict and "rnaNormal" in i_blatCoordinateDict[coordinate]): # for each coordinate, get a dict of corresponding blat hits blatHitsList = i_blatCoordinateDict[coordinate]["rnaNormal"] elif (modType == "SOM" or modType == "TUM_EDIT"): if (coordinate in i_blatCoordinateDict and "rnaTumor" in i_blatCoordinateDict[coordinate]): # for each coordinate, get a dict of corresponding blat hits blatHitsList = i_blatCoordinateDict[coordinate]["rnaTumor"] if (i_debug): logging.debug("coordinate=%s", coordinate) starts = 0 ends = 0 middles = 0 total = 0 # for each read, investigate the blat input for readId in blatHitsList: readIdList = readId.split("_") total += 1 position = readIdList[7] if (i_debug): logging.debug("readId=%s, position=%s", readId, position) if (position == "start"): starts += 1 elif (position == "end"): ends += 1 elif (position == "middle"): middles += 1 # if we have enough reads if (total > i_readDepthCutoff): if (round(starts/float(total),2) >= i_readPercentCutoff): modTypeFilters[modType] = "pbias" elif (round(ends/float(total),2) >= i_readPercentCutoff): modTypeFilters[modType] = "pbias" #elif (round(middles/float(total),2) >= i_readPercentCutoff): # modTypeFilters[modType] = "pbias" else: modTypeFilters[modType] = "PASS" atLeastOnePass = True # if we don't have the minimum number of reads, then pass else: modTypeFilters[modType] = "PASS" atLeastOnePass = True if (i_debug): logging.info("coordinate=%s, starts=%s, middles=%s, ends=%s, total=%s, positionalBias=%s", coordinate, starts, middles, ends, total, str(modTypeFilters)) # make a copy of the list to manipulate modTypesTmpList = list(modTypes) modChanges = vcfInfoDict["MC"] # if at least one passed, then remove the ones that didn't for (modType, modChange) in izip(modTypes, modChanges): # if at least one passed, then remove the ones that didn't if (atLeastOnePass): if (modTypeFilters[modType] == "pbias"): modTypesTmpList.remove(modType) modChanges.remove(modChange) # set the modTypes and modChanges vcfInfoDict["MT"] = modTypesTmpList vcfInfoDict["MC"] = modChanges # if at least one passed, then set pass if (not atLeastOnePass): vcfFilterSet.add("pbias") output = [vcfChr, str(vcfStopCoordinate), vcfId, vcfRef, vcfAlt, vcfScore] # if there are no filters so far, then this call passes if (len(vcfFilterSet) == 0): vcfFilterSet.add("PASS") output.append(";".join(vcfFilterSet)) # add the modified info dict infoField = "" for key in sorted(vcfInfoDict.iterkeys()): if (len(vcfInfoDict[key]) == 0): continue elif ("True" in vcfInfoDict[key]): infoField += key + ";" else: infoField += key + "=" + ",".join(vcfInfoDict[key]) + ";" output.append(infoField.rstrip(";")) output.append(restOfLine) if (i_outputFilename != None): i_outputFileHandler.write("\t".join(output) + "\n") else: print >> sys.stdout, "\t".join(output) # close the files if (i_outputFilename != None): i_outputFileHandler.close() return
def main(): # create the usage statement usage = "usage: python %prog id chrom vcfFile blatInputFile [Options]" i_cmdLineParser = OptionParser(usage=usage) i_cmdLineParser.add_option( "-o", "--outputFilename", dest="outputFilename", metavar="OUTPUT_FILE", help="the name of the output file, STDOUT by default") i_cmdLineParser.add_option( "-l", "--log", dest="logLevel", default="WARNING", metavar="LOG", help= "the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL), %default by default" ) i_cmdLineParser.add_option( "-g", "--logFilename", dest="logFilename", metavar="LOG_FILE", help="the name of the log file, STDOUT by default") i_cmdLineParser.add_option( "-d", "--readDepthCutoff", type="int", default=int(4), dest="readDepthCutoff", metavar="READ_DP_CUTOFF", help= "the minimum number of reads that are necessary before applying this filter, %default by default" ) i_cmdLineParser.add_option( "-p", "--readPercentCutoff", type="float", default=float(0.95), dest="readPercentCutoff", metavar="READ_PERCENT_CUTOFF", help= "the maximum percentage of reads with the alternative allele at the beginning or end of the reads, %default by default" ) i_cmdLineParser.add_option( "-k", "--keepPreviousFilters", action="store_true", default=False, dest="keepPreviousFilters", help= "by default the previous filters are overwritten with the pbias filter, include this argument if the previous filters should be kept" ) # range(inclusiveFrom, exclusiveTo, by) i_possibleArgLengths = range(4, 17, 1) i_argLength = len(sys.argv) # check if this is one of the possible correct commands if (i_argLength not in i_possibleArgLengths): i_cmdLineParser.print_help() sys.exit(1) # get the required parameters (i_cmdLineOptions, i_cmdLineArgs) = i_cmdLineParser.parse_args() i_id = str(i_cmdLineArgs[0]) i_vcfFilename = str(i_cmdLineArgs[1]) i_blatInputFilename = str(i_cmdLineArgs[2]) # get the optional params with default values i_logLevel = i_cmdLineOptions.logLevel i_readDepthCutoff = i_cmdLineOptions.readDepthCutoff i_readPercentCutoff = i_cmdLineOptions.readPercentCutoff i_keepPreviousFiltersFlag = i_cmdLineOptions.keepPreviousFilters # try to get any optional parameters with no defaults i_outputFilename = None i_logFilename = None if (i_cmdLineOptions.outputFilename != None): i_outputFilename = str(i_cmdLineOptions.outputFilename) if (i_cmdLineOptions.logFilename != None): i_logFilename = str(i_cmdLineOptions.logFilename) # assuming loglevel is bound to the string value obtained from the # command line argument. Convert to upper case to allow the user to # specify --log=DEBUG or --log=debug i_numericLogLevel = getattr(logging, i_logLevel.upper(), None) if not isinstance(i_numericLogLevel, int): raise ValueError( "Invalid log level: '%s' must be one of the following: DEBUG, INFO, WARNING, ERROR, CRITICAL", i_logLevel) # set up the logging if (i_logFilename != None): logging.basicConfig(level=i_numericLogLevel, filename=i_logFilename, filemode='w', format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') else: logging.basicConfig(level=i_numericLogLevel, format='%(asctime)s\t%(levelname)s\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') # set the debug i_debug = (i_numericLogLevel == logging.DEBUG) # output some debug info if (i_debug): logging.debug("id=%s", i_id) logging.debug("vcfFilename=%s", i_vcfFilename) logging.debug("blatInputFilename=%s", i_blatInputFilename) logging.debug("outputFilename=%s", i_outputFilename) logging.debug("logFilename=%s", i_logFilename) logging.debug("readDepthCutoff=%s", i_readDepthCutoff) logging.debug("readPerentCutoff=%s", i_readPercentCutoff) logging.debug("keepPreviousFiltersFlag? %s", i_keepPreviousFiltersFlag) # check for any errors i_writeFilenameList = [] if (i_outputFilename != None): i_writeFilenameList = [i_outputFilename] if (i_logFilename != None): i_writeFilenameList = [i_logFilename] i_readFilenameList = [i_vcfFilename, i_blatInputFilename] if (not radiaUtil.check_for_argv_errors(None, i_readFilenameList, i_writeFilenameList)): sys.exit(1) # open the output stream i_outputFileHandler = None if (i_outputFilename != None): i_outputFileHandler = get_write_fileHandler(i_outputFilename) # get the BLAT input i_blatCoordinateDict = parse_blat_input(i_blatInputFilename, i_debug) # get the VCF generator i_vcfGenerator = get_vcf_data(i_vcfFilename, i_debug) for (vcfChr, vcfStopCoordinate, vcfId, vcfRef, vcfAlt, vcfScore, vcfFilterSet, vcfInfoDict, restOfLine) in i_vcfGenerator: if (i_debug): logging.debug("VCF Data: %s %s %s %s %s %s %s %s %s", vcfChr, str(vcfStopCoordinate), vcfId, vcfRef, vcfAlt, vcfScore, str(vcfFilterSet), str(vcfInfoDict), restOfLine) modTypes = vcfInfoDict["MT"] modTypeFilters = dict() atLeastOnePass = False blatHitsList = list() # get the coordinate for this position coordinate = vcfChr + "_" + str(vcfStopCoordinate) for modType in modTypes: if (modType == "NOR_EDIT"): if (coordinate in i_blatCoordinateDict and "rnaNormal" in i_blatCoordinateDict[coordinate]): # for each coordinate, get a dict of corresponding blat hits blatHitsList = i_blatCoordinateDict[coordinate][ "rnaNormal"] elif (modType == "SOM" or modType == "TUM_EDIT"): if (coordinate in i_blatCoordinateDict and "rnaTumor" in i_blatCoordinateDict[coordinate]): # for each coordinate, get a dict of corresponding blat hits blatHitsList = i_blatCoordinateDict[coordinate]["rnaTumor"] if (i_debug): logging.debug("coordinate=%s", coordinate) starts = 0 ends = 0 middles = 0 total = 0 # for each read, investigate the blat input for readId in blatHitsList: readIdList = readId.split("_") total += 1 position = readIdList[7] if (i_debug): logging.debug("readId=%s, position=%s", readId, position) if (position == "start"): starts += 1 elif (position == "end"): ends += 1 elif (position == "middle"): middles += 1 # if we have enough reads if (total > i_readDepthCutoff): if (round(starts / float(total), 2) >= i_readPercentCutoff): modTypeFilters[modType] = "pbias" elif (round(ends / float(total), 2) >= i_readPercentCutoff): modTypeFilters[modType] = "pbias" #elif (round(middles/float(total),2) >= i_readPercentCutoff): # modTypeFilters[modType] = "pbias" else: modTypeFilters[modType] = "PASS" atLeastOnePass = True # if we don't have the minimum number of reads, then pass else: modTypeFilters[modType] = "PASS" atLeastOnePass = True if (i_debug): logging.debug( "coordinate=%s, starts=%s, middles=%s, ends=%s, total=%s, positionalBias=%s", coordinate, starts, middles, ends, total, str(modTypeFilters)) # make a copy of the list to manipulate modTypesTmpList = list(modTypes) modChanges = vcfInfoDict["MC"] # if at least one passed, then remove the ones that didn't for (modType, modChange) in izip(modTypes, modChanges): # if at least one passed, then remove the ones that didn't if (atLeastOnePass): if (modTypeFilters[modType] == "pbias"): modTypesTmpList.remove(modType) modChanges.remove(modChange) # set the modTypes and modChanges vcfInfoDict["MT"] = modTypesTmpList vcfInfoDict["MC"] = modChanges # if at least one passed, then set pass if (atLeastOnePass): vcfFilterSet = ["PASS"] else: # if the user wants to keep the previous filters if (i_keepPreviousFiltersFlag): # if the call previously passed, then just set pbias if (len(vcfFilterSet) == 1 and "PASS" in vcfFilterSet): vcfFilterSet = ["pbias"] # otherwise, add it to the previous filters else: vcfFilterSet.add("pbias") # otherwise, just set the pbias filter else: vcfFilterSet = ["pbias"] # update the mod filters modTypes = vcfInfoDict["MT"] modChanges = vcfInfoDict["MC"] origins = vcfInfoDict["ORIGIN"] modFilters = [] if vcfInfoDict["MF"] is None else vcfInfoDict["MF"] modFilterTypes = [] if vcfInfoDict["MFT"] is None else vcfInfoDict[ "MFT"] for origin in origins: for (modType, modChange) in izip(modTypes, modChanges): modFilterTypes.append("_".join( [origin, modType, modChange])) modFilters.append("_".join(vcfFilterSet)) vcfInfoDict["MF"] = modFilters vcfInfoDict["MFT"] = modFilterTypes output = [ vcfChr, str(vcfStopCoordinate), vcfId, vcfRef, vcfAlt, vcfScore ] # if there are no filters so far, then this call passes if (len(vcfFilterSet) == 0): vcfFilterSet.add("PASS") output.append(";".join(vcfFilterSet)) # add the modified info dict infoField = "" for key in sorted(vcfInfoDict.iterkeys()): if (len(vcfInfoDict[key]) == 0): continue elif ("True" in vcfInfoDict[key]): infoField += key + ";" else: infoField += key + "=" + ",".join(vcfInfoDict[key]) + ";" output.append(infoField.rstrip(";")) output.append(restOfLine) if (i_outputFilename != None): i_outputFileHandler.write("\t".join(output) + "\n") else: print >> sys.stdout, "\t".join(output) # close the files if (i_outputFilename != None): i_outputFileHandler.close() return