def processRead(input, args=False): base = os.path.basename(input).split('.')[0] PL = len(FwdPrimer) RL = len(RevPrimer) DemuxOut = os.path.join(tmpdir, base+'.demux.fq') StatsOut = os.path.join(tmpdir, base+'.stats') Total = 0 NoBarcode = 0 NoRevBarcode = 0 NoPrimer = 0 TooShort = 0 RevPrimerFound = 0 ValidSeqs = 0 RCrevPrimer = amptklib.RevComp(RevPrimer) with open(StatsOut, 'w') as counts: with open(DemuxOut, 'w') as out: for title, seq, qual in FastqGeneralIterator(open(input)): Total += 1 #look for barcode, trim it off Barcode, BarcodeLabel = amptklib.AlignBarcode(seq, Barcodes, args.barcode_mismatch) if Barcode == "": NoBarcode += 1 continue BarcodeLength = len(Barcode) Seq = seq[BarcodeLength:] Qual = qual[BarcodeLength:] #now search for forward primer foralign = edlib.align(FwdPrimer, Seq, mode="HW", k=args.primer_mismatch, additionalEqualities=amptklib.degenNuc) if foralign["editDistance"] < 0: NoPrimer += 1 continue ForTrim = foralign["locations"][0][1]+1 #now search for reverse primer revalign = edlib.align(RCrevPrimer, Seq, mode="HW", task="locations", k=args.primer_mismatch, additionalEqualities=amptklib.degenNuc) if revalign["editDistance"] >= 0: #reverse primer was found RevPrimerFound += 1 #location to trim sequences RevTrim = revalign["locations"][0][0] #determine reverse barcode if args.reverse_barcode: RevBCdiffs = 0 BCcut = revalign["locations"][0][1] CutSeq = Seq[BCcut:] RevBarcode, RevBarcodeLabel = amptklib.AlignRevBarcode(CutSeq, RevBarcodes, args.barcode_mismatch) if RevBarcode == "": NoRevBarcode += 1 continue BarcodeLabel = BarcodeLabel+':-:'+RevBarcodeLabel #now trim record remove forward and reverse reads Seq = Seq[ForTrim:RevTrim] Qual = Qual[ForTrim:RevTrim] #since found reverse primer, now also need to pad/trim if not args.full_length: #check minimum length here or primer dimer type sequences will get padded with Ns if len(Seq) < int(args.min_len): TooShort += 1 continue if len(Seq) < args.trim_len and args.pad == 'on': pad = args.trim_len - len(Seq) Seq = Seq + pad*'N' Qual = Qual +pad*'I' else: #len(Seq) > args.trim_len: Seq = Seq[:args.trim_len] Qual = Qual[:args.trim_len] else: #trim record, did not find reverse primer if args.full_length: #if full length then move to next record continue #trim away forward primer Seq = Seq[ForTrim:] Qual = Qual[ForTrim:] #check length and trim, throw away if too short as it was bad read if len(Seq) < args.trim_len: TooShort += 1 continue Seq = Seq[:args.trim_len] Qual = Qual[:args.trim_len] #check minimum length if len(Seq) < int(args.min_len): TooShort += 1 continue ValidSeqs += 1 #rename header Name = 'R_'+str(ValidSeqs)+';barcodelabel='+BarcodeLabel+';' out.write("@%s\n%s\n+\n%s\n" % (Name, Seq, Qual)) counts.write('%i,%i,%i,%i,%i,%i,%i\n' % (Total, NoBarcode, NoPrimer, RevPrimerFound, NoRevBarcode, TooShort, ValidSeqs))
def main(args): global FwdPrimer, RevPrimer, SampleData, Barcodes, RevBarcodes, tmpdir, usearch parser=argparse.ArgumentParser(prog='amptk-process_ion.py', usage="%(prog)s [options] -i file.fastq\n%(prog)s -h for help menu", description='''Script finds barcodes, strips forward and reverse primers, relabels, and then trim/pads reads to a set length''', epilog="""Written by Jon Palmer (2015) [email protected]""", formatter_class=MyFormatter) parser.add_argument('-i','--fastq', dest='fastq', required=True, help='FASTQ R1 file') parser.add_argument('--reverse', help='Illumina R2 reverse reads') parser.add_argument('-o','--out', dest="out", default='illumina2', help='Base name for output') parser.add_argument('-f','--fwd_primer', dest="F_primer", default='fITS7', help='Forward Primer') parser.add_argument('-r','--rev_primer', dest="R_primer", default='ITS4', help='Reverse Primer') parser.add_argument('-m','--mapping_file', help='Mapping file: QIIME format can have extra meta data columns') parser.add_argument('-p','--pad', default='off', choices=['on', 'off'], help='Pad with Ns to a set length') parser.add_argument('--primer_mismatch', default=2, type=int, help='Number of mis-matches in primer') parser.add_argument('--barcode_mismatch', default=0, type=int, help='Number of mis-matches in barcode') parser.add_argument('--barcode_fasta', help='FASTA file containing Barcodes (Names & Sequences)') parser.add_argument('--barcode_not_anchored', action='store_true', help='Barcodes (indexes) are not at start of reads') parser.add_argument('--reverse_barcode', help='FASTA file containing 3 prime Barocdes') parser.add_argument('--min_len', default=100, type=int, help='Minimum read length to keep') parser.add_argument('-l','--trim_len', default=300, type=int, help='Trim length for reads') parser.add_argument('--full_length', action='store_true', help='Keep only full length reads (no trimming/padding)') parser.add_argument('--merge_method', default='usearch', choices=['usearch', 'vsearch'], help='Software to use for PE read merging') parser.add_argument('--cpus', type=int, help="Number of CPUs. Default: auto") parser.add_argument('-u','--usearch', dest="usearch", default='usearch9', help='USEARCH EXE') args=parser.parse_args(args) args.out = re.sub(r'\W+', '', args.out) log_name = args.out + '.amptk-demux.log' if os.path.isfile(log_name): os.remove(log_name) FNULL = open(os.devnull, 'w') amptklib.setupLogging(log_name) cmd_args = " ".join(sys.argv)+'\n' amptklib.log.debug(cmd_args) print("-------------------------------------------------------") #initialize script, log system info and usearch version amptklib.SystemInfo() #Do a version check usearch = args.usearch amptklib.versionDependencyChecks(usearch) #get number of CPUs to use if not args.cpus: cpus = multiprocessing.cpu_count() else: cpus = args.cpus #parse a mapping file or a barcode fasta file, primers, etc get setup #dealing with Barcodes, get ion barcodes or parse the barcode_fasta argument barcode_file = args.out + ".barcodes_used.fa" rev_barcode_file = args.out + '.revbarcodes_used.fa' amptklib.SafeRemove(barcode_file) amptklib.SafeRemove(rev_barcode_file) #check if mapping file passed, use this if present, otherwise use command line arguments SampleData = {} Barcodes = {} RevBarcodes = {} FwdPrimer = '' RevPrimer = '' if args.mapping_file: if not os.path.isfile(args.mapping_file): amptklib.log.error("Mapping file not found: %s" % args.mapping_file) sys.exit(1) SampleData, Barcodes, RevBarcodes, FwdPrimer, RevPrimer = amptklib.parseMappingFileNEW(args.mapping_file) else: #no mapping file, so create dictionaries from barcode fasta files if not args.barcode_fasta: amptklib.log.error("You did not specify a --barcode_fasta or --mapping_file, one is required") sys.exit(1) else: shutil.copyfile(args.barcode_fasta, barcode_file) Barcodes = amptklib.fasta2barcodes(barcode_file, False) if args.reverse_barcode: shutil.copyfile(args.reverse_barcode, rev_barcode_file) RevBarcodes = amptklib.fasta2barcodes(rev_barcode_file, False) #parse primers here so doesn't conflict with mapping primers #look up primer db otherwise default to entry if args.F_primer in amptklib.primer_db: FwdPrimer = amptklib.primer_db.get(args.F_primer) amptklib.log.info("{:} fwd primer found in AMPtk primer db, setting to: {:}".format(args.F_primer, FwdPrimer)) else: FwdPrimer = args.F_primer amptklib.log.info("{:} fwd primer not found in AMPtk primer db, assuming it is actual primer sequence.".format(args.F_primer)) if args.R_primer in amptklib.primer_db: RevPrimer = amptklib.primer_db.get(args.R_primer) amptklib.log.info("{:} rev primer found in AMPtk primer db, setting to: {:}".format(args.R_primer, RevPrimer)) else: RevPrimer = args.R_primer amptklib.log.info("{:} rev primer not found in AMPtk primer db, assuming it is actual primer sequence.".format(args.R_primer)) #check if input is compressed gzip_list = [] if args.fastq.endswith('.gz'): gzip_list.append(os.path.abspath(args.fastq)) if args.reverse: if args.reverse.endswith('.gz'): gzip_list.append(os.path.abspath(args.reverse)) if gzip_list: amptklib.log.info("Gzipped input files detected, uncompressing") for file in gzip_list: file_out = file.replace('.gz', '') amptklib.Funzip(file, file_out, cpus) args.fastq = args.fastq.replace('.gz', '') if args.reverse: args.reverse = args.reverse.replace('.gz', '') #Count FASTQ records amptklib.log.info("Loading FASTQ Records") orig_total = amptklib.countfastq(args.fastq) size = amptklib.checkfastqsize(args.fastq) readablesize = amptklib.convertSize(size*2) amptklib.log.info('{:,} reads ({:})'.format(orig_total, readablesize)) #output barcodes/samples amptklib.log.info('Searching for {:} forward barcodes and {:} reverse barcodes'.format(len(Barcodes), len(RevBarcodes))) #create tmpdir and split input into n cpus tmpdir = args.out.split('.')[0]+'_'+str(os.getpid()) if not os.path.exists(tmpdir): os.makedirs(tmpdir) #tell user about number of cores using amptklib.log.info("Splitting FASTQ files over {:} cpus".format(cpus)) if args.reverse: amptklib.log.info("Demuxing PE Illumina reads; FwdPrimer: {:} RevPrimer: {:}".format(FwdPrimer, RevPrimer)) else: amptklib.log.info("Demuxing SE Illumina reads; FwdPrimer: {:} RevPrimer: {:}".format(FwdPrimer, amptklib.RevComp(RevPrimer))) amptklib.log.info('Dropping reads less than {:} bp and setting lossless trimming to {:} bp.'.format(args.min_len, args.trim_len)) if cpus > 1: if args.reverse: amptklib.split_fastqPE(args.fastq, args.reverse, orig_total, tmpdir, cpus*4) file_list = [] for file in os.listdir(tmpdir): if file.endswith('.fq'): filepart = os.path.join(tmpdir, file.split('_R')[0]) if not filepart in file_list: file_list.append(filepart) amptklib.runMultiProgress(processReadsPE, file_list, cpus, args=args) else: #split fastq file amptklib.split_fastq(args.fastq, orig_total, tmpdir, cpus*4) #now get file list from tmp folder file_list = [] for file in os.listdir(tmpdir): if file.endswith(".fq"): file = os.path.join(tmpdir, file) file_list.append(file) #finally process reads over number of cpus amptklib.runMultiProgress(processRead, file_list, cpus, args=args) else: if args.reverse: shutil.copyfile(args.fastq, os.path.join(tmpdir, 'chunk_R1.fq')) shutil.copyfile(args.reverse, os.path.join(tmpdir, 'chunk_R2.fq')) processReadsPE(os.path.join(tmpdir, 'chunk'), args=args) else: shutil.copyfile(args.fastq, os.path.join(tmpdir, 'chunk.fq')) processRead(os.path.join(tmpdir, 'chunk.fq'), args=args) print("-------------------------------------------------------") #Now concatenate all of the demuxed files together amptklib.log.info("Concatenating Demuxed Files") tmpDemux = args.out + '.tmp.demux.fq' with open(tmpDemux, 'w') as outfile: for filename in glob.glob(os.path.join(tmpdir,'*.demux.fq')): if filename == tmpDemux: continue with open(filename, 'r') as readfile: shutil.copyfileobj(readfile, outfile) if args.reverse: #parse the stats finalstats = [0,0,0,0,0,0] for file in os.listdir(tmpdir): if file.endswith('.stats'): with open(os.path.join(tmpdir, file), 'r') as statsfile: line = statsfile.readline() line = line.rstrip() newstats = line.split(',') newstats = [int(i) for i in newstats] for x, num in enumerate(newstats): finalstats[x] += num amptklib.log.info('{0:,}'.format(finalstats[0])+' total reads') amptklib.log.info('{0:,}'.format(finalstats[0]-finalstats[1]-finalstats[3])+' valid Barcodes') amptklib.log.info('{0:,}'.format(finalstats[5])+' valid output reads (Barcodes and Primers)') else: #parse the stats finalstats = [0,0,0,0,0,0,0] for file in os.listdir(tmpdir): if file.endswith('.stats'): with open(os.path.join(tmpdir, file), 'r') as statsfile: line = statsfile.readline() line = line.rstrip() newstats = line.split(',') newstats = [int(i) for i in newstats] for x, num in enumerate(newstats): finalstats[x] += num amptklib.log.info('{0:,}'.format(finalstats[0])+' total reads') if args.reverse_barcode: amptklib.log.info('{0:,}'.format(finalstats[0]-finalstats[1]-finalstats[2]-finalstats[4])+' valid Fwd and Rev Barcodes') else: amptklib.log.info('{0:,}'.format(finalstats[0]-finalstats[1])+' valid Barcode') amptklib.log.info('{0:,}'.format(finalstats[0]-finalstats[1]-finalstats[2])+' Fwd Primer found, {0:,}'.format(finalstats[3])+ ' Rev Primer found') amptklib.log.info('{0:,}'.format(finalstats[5])+' discarded too short (< %i bp)' % args.min_len) amptklib.log.info('{0:,}'.format(finalstats[6])+' valid output reads') #clean up tmp folder amptklib.SafeRemove(tmpdir) #last thing is to re-number of reads as it is possible they could have same name from multitprocessor split catDemux = args.out+'.demux.fq' amptklib.fastqreindex(tmpDemux, catDemux) amptklib.SafeRemove(tmpDemux) #now loop through data and find barcoded samples, counting each..... BarcodeCount = {} with open(catDemux, 'r') as input: header = itertools.islice(input, 0, None, 4) for line in header: ID = line.split("=",1)[-1].split(";")[0] if ID not in BarcodeCount: BarcodeCount[ID] = 1 else: BarcodeCount[ID] += 1 #now let's count the barcodes found and count the number of times they are found. barcode_counts = "%22s: %s" % ('Sample', 'Count') for k,v in natsorted(list(BarcodeCount.items()), key=lambda k_v: k_v[1], reverse=True): barcode_counts += "\n%22s: %s" % (k, str(BarcodeCount[k])) amptklib.log.info("Found %i barcoded samples\n%s" % (len(BarcodeCount), barcode_counts)) genericmapfile = args.out + '.mapping_file.txt' if not args.mapping_file: #create a generic mappingfile for downstream processes amptklib.CreateGenericMappingFile(Barcodes, RevBarcodes, FwdPrimer, RevPrimer, genericmapfile, BarcodeCount) else: amptklib.updateMappingFile(args.mapping_file, BarcodeCount, genericmapfile) #compress the output to save space FinalDemux = catDemux+'.gz' amptklib.Fzip(catDemux, FinalDemux, cpus) amptklib.removefile(catDemux) if gzip_list: for file in gzip_list: file = file.replace('.gz', '') amptklib.removefile(file) #get file size filesize = os.path.getsize(FinalDemux) readablesize = amptklib.convertSize(filesize) amptklib.log.info("Output file: %s (%s)" % (FinalDemux, readablesize)) amptklib.log.info("Mapping file: %s" % genericmapfile) print("-------------------------------------------------------") if 'darwin' in sys.platform: print(col.WARN + "\nExample of next cmd: " + col.END + "amptk cluster -i %s -o out\n" % (FinalDemux)) else: print("\nExample of next cmd: amptk cluster -i %s -o out\n" % (FinalDemux))
def processSEreads(input, args=False): #input is expected to be a FASTQ file #local variables that need to be previously declared: FwdPrimer, RevPrimer inputPath = os.path.abspath(input) Name = os.path.basename(inputPath).split(".fq", -1)[0] DemuxOut = os.path.join(args.out, Name + '.demux.fq') Sample = Name.split('_')[0] StatsOut = os.path.join(args.out, Name + '.stats') Total = 0 NoPrimer = 0 TooShort = 0 RevPrimerFound = 0 ValidSeqs = 0 multiHits = 0 RCRevPrimer = amptklib.RevComp(RevPrimer) with open(StatsOut, 'w') as counts: with open(DemuxOut, 'w') as out: for title, seq, qual in FastqGeneralIterator(open(inputPath)): Total += 1 #first thing is look for forward primer, if found trim it off foralign = edlib.align(FwdPrimer, seq, mode="HW", k=args.primer_mismatch, additionalEqualities=amptklib.degenNuc) #if require primer is on make finding primer in amplicon required if amplicon is larger than read length #if less than read length, can't enforce primer because could have been trimmed via staggered trim in fastq_mergepairs if len(foralign['locations']) > 1: multiHits += 1 continue if args.primer == 'on': if foralign["editDistance"] < 0: NoPrimer += 1 continue ForCutPos = foralign["locations"][0][1] + 1 Seq = seq[ForCutPos:] Qual = qual[ForCutPos:] else: if foralign["editDistance"] >= 0: ForCutPos = foralign["locations"][0][1] + 1 Seq = seq[ForCutPos:] Qual = qual[ForCutPos:] else: NoPrimer += 1 Seq = seq Qual = qual #now look for reverse primer revalign = edlib.align(RCRevPrimer, Seq, mode="HW", task="locations", k=args.primer_mismatch, additionalEqualities=amptklib.degenNuc) if revalign["editDistance"] >= 0: RevPrimerFound += 1 RevCutPos = revalign["locations"][0][0] #location to trim sequences, trim seqs Seq = Seq[:RevCutPos] Qual = Qual[:RevCutPos] else: if args.full_length: continue #if full_length is passed, then only trim primers if not args.full_length: #got here if primers were found they were trimmed #now check seq length, pad if too short, trim if too long if len( Seq ) < args.min_len: #need this check here or primer dimers will get through TooShort += 1 continue if len(Seq) < args.trim_len and args.pad == 'on': pad = args.trim_len - len(Seq) Seq = Seq + pad * 'N' Qual = Qual + pad * 'I' else: #len(Seq) > args.trim_len: Seq = Seq[:args.trim_len] Qual = Qual[:args.trim_len] #got here, reads are primers trimmed and trim/padded, check length if len(Seq) < args.min_len: TooShort += 1 continue ValidSeqs += 1 #now fix header Title = 'R_' + str(ValidSeqs) + ';barcodelabel=' + Sample + ';' #now write to file out.write("@%s\n%s\n+\n%s\n" % (Title, Seq, Qual)) ForPrimerFound = Total - NoPrimer - multiHits counts.write("%i,%i,%i,%i,%i,%i\n" % (Total, ForPrimerFound, RevPrimerFound, multiHits, TooShort, ValidSeqs))
def main(args): parser=argparse.ArgumentParser(prog='amptk-fastq2sra.py', usage="%(prog)s [options] -i folder", description='''Script to split FASTQ file from Ion, 454, or Illumina by barcode sequence into separate files for submission to SRA. This script can take the BioSample worksheet from NCBI and create an SRA metadata file for submission.''', epilog="""Written by Jon Palmer (2015) [email protected]""", formatter_class=MyFormatter) parser.add_argument('-i','--input', dest='FASTQ', required=True, help='Input FASTQ file or folder') parser.add_argument('-o','--out', dest='out', help='Basename for output folder/files') parser.add_argument('--min_len', default=50, type=int, help='Minimum length of read to keep') parser.add_argument('-b','--barcode_fasta', help='Multi-fasta file containing barcodes used') parser.add_argument('--reverse_barcode', help='Reverse barcode fasta file') parser.add_argument('-s','--biosample', dest='biosample', help='BioSample file from NCBI') parser.add_argument('-p','--platform', dest='platform', default='ion', choices=['ion', 'illumina', '454'], help='Sequencing platform') parser.add_argument('-f','--fwd_primer', dest="F_primer", default='fITS7', help='Forward Primer (fITS7)') parser.add_argument('-r','--rev_primer', dest="R_primer", default='ITS4', help='Reverse Primer (ITS4)') parser.add_argument('-n', '--names', help='CSV mapping file BC,NewName') parser.add_argument('-d', '--description', help='Paragraph description for SRA metadata') parser.add_argument('-t','--title', default='Fungal ITS', help='Start of title for SRA submission, name it according to amplicon') parser.add_argument('-m','--mapping_file', help='Mapping file: QIIME format can have extra meta data columns') parser.add_argument('--primer_mismatch', default=2, type=int, help='Number of mis-matches in primer') parser.add_argument('--barcode_mismatch', default=0, type=int, help='Number of mis-matches in barcode') parser.add_argument('--require_primer', default='off', choices=['forward', 'both', 'off'], help='Require Primers to be present') parser.add_argument('--force', action='store_true', help='Overwrite existing directory') parser.add_argument('-a','--append', help='Append a name to all sample names for a run, i.e. --append run1 would yield Sample_run1') args=parser.parse_args(args) #get basename if not args.out passed if args.out: base = args.out else: if 'demux' in args.FASTQ: base = os.path.basename(args.FASTQ).split('.demux')[0] else: base = os.path.basename(args.FASTQ).split('.f')[0] log_name = base + '.amptk-sra.log' if os.path.isfile(log_name): os.remove(log_name) amptklib.setupLogging(log_name) FNULL = open(os.devnull, 'w') cmd_args = " ".join(sys.argv)+'\n' amptklib.log.debug(cmd_args) print("-------------------------------------------------------") amptklib.SystemInfo() amptkversion = amptklib.get_version() #create output directory if not os.path.exists(base): os.makedirs(base) else: if not args.force: amptklib.log.error("Directory %s exists, add --force argument to overwrite" % base) sys.exit(1) else: shutil.rmtree(base) os.makedirs(base) #parse a mapping file or a barcode fasta file, primers, etc get setup #dealing with Barcodes, get ion barcodes or parse the barcode_fasta argument barcode_file = os.path.join(base, base + ".barcodes_used.fa") rev_barcode_file = os.path.join(base, base + ".revbarcodes_used.fa") if os.path.isfile(barcode_file): os.remove(barcode_file) #check if mapping file passed, use this if present, otherwise use command line arguments SampleData = {} Barcodes = {} RevBarcodes = {} FwdPrimer = '' RevPrimer = '' if args.mapping_file: if not os.path.isfile(args.mapping_file): amptklib.log.error("Mapping file not found: %s" % args.mapping_file) sys.exit(1) SampleData, Barcodes, RevBarcodes, FwdPrimer, RevPrimer = amptklib.parseMappingFileNEW(args.mapping_file) else: if args.barcode_fasta: with open(barcode_file, 'w') as barcodeout: with open(args.barcode_fasta, 'r') as input: for rec in SeqIO.parse(input, 'fasta'): outname = args.multi+'.'+rec.id barcodeout.write(">%s\n%s\n" % (outname, rec.seq)) if args.reverse_barcode: with open(rev_barcode_file, 'w') as barcodeout: with open(args.reverse_barcode, 'r') as input: for rec in SeqIO.parse(input, 'fasta'): outname = args.multi+'.'+rec.id barcodeout.write(">%s\n%s\n" % (outname, rec.seq)) #parse primers here so doesn't conflict with mapping primers #look up primer db otherwise default to entry if FwdPrimer == '': if args.F_primer in amptklib.primer_db: FwdPrimer = amptklib.primer_db.get(args.F_primer) amptklib.log.info("{:} fwd primer found in AMPtk primer db, setting to: {:}".format(args.F_primer, FwdPrimer)) else: FwdPrimer = args.F_primer amptklib.log.info("{:} fwd primer not found in AMPtk primer db, assuming it is actual primer sequence.".format(args.F_primer)) if RevPrimer == '': if args.R_primer in amptklib.primer_db: RevPrimer = amptklib.primer_db.get(args.R_primer) amptklib.log.info("{:} rev primer found in AMPtk primer db, setting to: {:}".format(args.R_primer, RevPrimer)) else: RevPrimer = args.R_primer amptklib.log.info("{:} rev primer not found in AMPtk primer db, assuming it is actual primer sequence.".format(args.R_primer)) #then setup barcode dictionary if len(Barcodes) < 1 and os.path.isfile(barcode_file): Barcodes = amptklib.fasta2barcodes(barcode_file, False) #setup for looking for reverse barcode if len(RevBarcodes) < 1 and args.reverse_barcode: if not os.path.isfile(args.reverse_barcode): amptklib.log.info("Reverse barcode is not a valid file, exiting") sys.exit(1) shutil.copyfile(args.reverse_barcode, rev_barcode_file) RevBarcodes = amptklib.fasta2barcodes(rev_barcode_file, True) if args.platform != 'illumina': if not args.mapping_file and not args.barcode_fasta: amptklib.log.error("For ion, 454, or illumina2 datasets you must specificy a multi-fasta file containing barcodes with -b, --barcode_fasta, or -m/--mapping_file") sys.exit(1) if args.platform == 'illumina': #just need to get the correct .fastq.gz files into a folder by themselves #if illumina is selected, verify that args.fastq is a folder if not os.path.isdir(args.FASTQ): amptklib.log.error("%s is not a folder, for '--platform illumina', -i must be a folder containing raw reads" % (args.FASTQ)) sys.exit(1) rawlist = [] filelist = [] for file in os.listdir(args.FASTQ): if file.endswith(".fastq.gz") or file.endswith('.fastq') or file.endswith('.fq'): rawlist.append(file) if len(rawlist) > 0: if not '_R2' in sorted(rawlist)[1]: amptklib.log.info("Found %i single files, copying to %s folder" % (len(rawlist), base)) filelist = rawlist for file in rawlist: shutil.copyfile(os.path.join(args.FASTQ,file),(os.path.join(base,file))) else: amptklib.log.info("Found %i paired-end files, copying to %s folder" % (len(rawlist) / 2, base)) for file in rawlist: shutil.copyfile(os.path.join(args.FASTQ,file),(os.path.join(base,file))) if '_R1' in file: filelist.append(file) else: #start here to process the reads, first reverse complement the reverse primer ReverseCompRev = amptklib.RevComp(RevPrimer) #if --names given, load into dictonary if args.names: amptklib.log.info("Parsing names for output files via %s" % args.names) namesDict = {} with open(args.names, 'r') as input: for line in input: line = line.replace('\n', '') cols = line.split(',') if not cols[0] in namesDict: namesDict[cols[0]] = cols[1] #check for compressed input file if args.FASTQ.endswith('.gz'): amptklib.log.info("Gzipped input files detected, uncompressing") FASTQ_IN = args.FASTQ.replace('.gz', '') amptklib.Funzip(args.FASTQ, FASTQ_IN, multiprocessing.cpu_count()) else: FASTQ_IN = args.FASTQ #count FASTQ records in input amptklib.log.info("Loading FASTQ Records") total = amptklib.countfastq(FASTQ_IN) size = amptklib.checkfastqsize(args.FASTQ) readablesize = amptklib.convertSize(size) amptklib.log.info('{0:,}'.format(total) + ' reads (' + readablesize + ')') #output message depending on primer requirement if args.require_primer == 'off': amptklib.log.info("Looking for %i barcodes" % (len(Barcodes))) elif args.require_primer == 'forward': amptklib.log.info("Looking for %i barcodes that must have FwdPrimer: %s" % (len(Barcodes), FwdPrimer)) elif args.require_primer == 'both': amptklib.log.info("Looking for %i barcodes that must have FwdPrimer: %s and RevPrimer: %s" % (len(Barcodes), FwdPrimer, RevPrimer)) #this will loop through FASTQ file once, splitting those where barcodes are found, and primers trimmed runningTotal = 0 with open(FASTQ_IN, 'r') as input: for title, seq, qual in FastqGeneralIterator(input): Barcode, BarcodeLabel = amptklib.AlignBarcode(seq, Barcodes, args.barcode_mismatch) if Barcode == "": continue #trim barcode from sequence BarcodeLength = len(Barcode) seq = seq[BarcodeLength:] qual = qual[BarcodeLength:] #look for forward primer if args.require_primer != 'off': #means we only want ones with forward primer and or reverse, but don't remove them #now search for forward primer foralign = edlib.align(FwdPrimer, seq, mode="HW", k=args.primer_mismatch, additionalEqualities=amptklib.degenNuc) if foralign["editDistance"] < 0: continue if args.require_primer == 'both': #now search for reverse primer revalign = edlib.align(ReverseCompRev, seq, mode="HW", task="locations", k=args.primer_mismatch, additionalEqualities=amptklib.degenNuc) if revalign["editDistance"] < 0: #reverse primer was not found continue #check size if len(seq) < args.min_len: #filter out sequences less than minimum length. continue runningTotal += 1 fileout = os.path.join(base, BarcodeLabel+'.fastq') with open(fileout, 'a') as output: output.write("@%s\n%s\n+\n%s\n" % (title, seq, qual)) if args.require_primer == 'off': amptklib.log.info('{0:,}'.format(runningTotal) + ' total reads with valid barcode') elif args.require_primer == 'forward': amptklib.log.info('{0:,}'.format(runningTotal) + ' total reads with valid barcode and fwd primer') elif args.require_primer == 'both': amptklib.log.info('{0:,}'.format(runningTotal) + ' total reads with valid barcode and both primers') amptklib.log.info("Now Gzipping files") for file in os.listdir(base): if file.endswith(".fastq"): file_path = os.path.join(base, file) amptklib.Fzip_inplace(file_path) #after all files demuxed into output folder, loop through and create SRA metadata file filelist = [] for file in os.listdir(base): if file.endswith(".fastq.gz"): filelist.append(file) amptklib.log.info("Finished: output in %s" % base) #clean up if gzipped if args.FASTQ.endswith('.gz'): amptklib.removefile(FASTQ_IN) #check for BioSample meta file if args.biosample: amptklib.log.info("NCBI BioSample file detected, creating SRA metadata file") #load in BioSample file to dictionary with open(args.biosample, 'r') as input: reader = csv.reader(input, delimiter=str('\t')) header = next(reader) acc = header.index('Accession') sample = header.index('Sample Name') bio = header.index('BioProject') try: host = header.index('Host') except ValueError: host = header.index('Organism') BioDict = {col[sample]:(col[acc],col[bio],col[host]) for col in reader} #set some defaults based on the platform header = 'bioproject_accession\tbiosample_accession\tlibrary_ID\ttitle\tlibrary_strategy\tlibrary_source\tlibrary_selection\tlibrary_layout\tplatform\tinstrument_model\tdesign_description\tfiletype\tfilename\tfilename2\tforward_barcode\treverse_barcode\tforward_primer\treverse_primer\n' if args.platform == 'ion': sequencer = 'ION_TORRENT' model = 'Ion Torrent PGM' lib_layout = 'single' elif args.platform == '454': sequencer = '_LS454' model = '454 GS FLX Titanium' lib_layout = 'single' elif args.platform == 'illumina': sequencer = 'ILLUMINA' model = 'Illumina MiSeq' lib_layout = 'paired' else: amptklib.log.error("You specified a platform that is not supported") sys.exit(1) lib_strategy = 'AMPLICON' lib_source = 'GENOMIC' lib_selection = 'RANDOM PCR' filetype = 'fastq' #now open file for writing, input header and then loop through samples sub_out = base + '.submission.txt' with open(sub_out, 'w') as output: output.write(header) for file in filelist: barcode_for = '' barcode_rev = '' if not args.description: description = '%s amplicon library was created using a barcoded fusion primer PCR protocol using Pfx50 polymerase (Thermo Fisher Scientific), size selected, and sequenced on the %s platform. Sequence data was minimally processed, sequences were exported directly from the sequencing platform and only the barcode (index sequence) was trimmed prior to SRA submission. SRA submission generated with AMPtk %s' % (args.title, model, amptkversion.split(' ')[-1]) else: description = args.description if args.platform == 'ion' or args.platform == '454': name = file.split(".fastq")[0] if not name in BioDict: #lets try to look a bit harder, i.e. split on _ and - and look again searchname = name.replace('-', '_') searchname = searchname.split('_')[0] if not searchname in BioDict: #if still not found, then skip continue else: searchname = name bioproject = BioDict.get(searchname)[1] if not bioproject.startswith('PRJNA'): bioproject = 'PRJNA'+bioproject sample_name = BioDict.get(searchname)[0] title = '%s amplicon sequencing of %s: sample %s' % (args.title, BioDict.get(name)[2], name) bc_name = file.split(".f")[0] if bc_name in Barcodes: barcode_for = Barcodes.get(bc_name) if bc_name in RevBarcodes: barcode_rev = RevBarcodes.get(bc_name) if args.append: finalname = name+'_'+args.append #also need to change the name for output files newfile = file.replace(name, finalname) os.rename(os.path.join(base, file), os.path.join(base, newfile)) else: finalname = name newfile = file line = [bioproject,sample_name,finalname,title,lib_strategy,lib_source,lib_selection,lib_layout,sequencer,model,description,filetype,newfile,'',barcode_for,barcode_rev,FwdPrimer,RevPrimer] elif args.platform == 'illumina': name = file.split("_")[0] if not name in BioDict: amptklib.log.info('{:} not found in BioSample text file'.format(name)) continue bioproject = BioDict.get(name)[1] if not bioproject.startswith('PRJNA'): bioproject = 'PRJNA'+bioproject sample_name = BioDict.get(name)[0] title = '%s amplicon sequencing of %s: sample %s' % (args.title, BioDict.get(name)[2], name) file2 = file.replace('_R1', '_R2') #count number of _ in name, determines the dataformat fields = file.count("_") if fields > 3: #this is full illumina name with dual barcodes dualBC = file.split("_")[1] if '-' in dualBC: barcode_for = dualBC.split('-')[0] barcode_rev = dualBC.split('-')[1] elif fields == 3: #this is older reverse barcoded name barcode_for = '' barcode_rev = file.split("_")[1] if args.append: finalname = name+'_'+args.append newfile = file.replace(name, finalname) newfile2 = file2.replace(name, finalname) #also need to change the name for output files os.rename(os.path.join(base, file), os.path.join(base, newfile1)) os.rename(os.path.join(base, file2), os.path.join(base, newfile2)) file = file.replace(name, finalname) else: finalname = name newfile = file newfile2 = file2 line = [bioproject,sample_name,finalname,title,lib_strategy,lib_source,lib_selection,lib_layout,sequencer,model,description,filetype,newfile,newfile2,barcode_for,barcode_rev,FwdPrimer,RevPrimer] #write output to file output.write('\t'.join(line)+'\n') amptklib.log.info("SRA submission file created: %s" % sub_out)
def main(args): global FwdPrimer, RevPrimer, Barcodes, tmpdir, usearch parser = argparse.ArgumentParser( prog='amptk-process_illumina_raw.py', usage="%(prog)s [options] -i file.fastq\n%(prog)s -h for help menu", description= '''Script finds barcodes, strips forward and reverse primers, relabels, and then trim/pads reads to a set length''', epilog="""Written by Jon Palmer (2015) [email protected]""", formatter_class=MyFormatter) parser.add_argument('-f', '--forward', dest='fastq', required=True, help='Illumina FASTQ R1 reads') parser.add_argument('-r', '--reverse', required=True, help='Illumina FASTQ R2 reads') parser.add_argument('-i', '--index', nargs='+', required=True, help='Illumina FASTQ index reads') parser.add_argument('-m', '--mapping_file', help='QIIME-like mapping file') parser.add_argument('--read_length', type=int, help='Read length, i.e. 2 x 300 bp = 300') parser.add_argument('-o', '--out', dest="out", default='illumina_out', help='Base name for output') parser.add_argument('--fwd_primer', dest="F_primer", default='515FB', help='Forward Primer') parser.add_argument('--rev_primer', dest="R_primer", default='806RB', help='Reverse Primer') parser.add_argument('--primer_mismatch', default=2, type=int, help='Number of mis-matches in primer') parser.add_argument('--barcode_mismatch', default=0, type=int, help='Number of mis-matches in barcode') parser.add_argument( '--barcode_fasta', help='FASTA file containing Barcodes (Names & Sequences)') parser.add_argument('--rescue_forward', default='on', choices=['on', 'off'], help='Rescue Not-merged forward reads') parser.add_argument('--barcode_rev_comp', action='store_true', help='Reverse complement barcode sequences') parser.add_argument('--min_len', default=100, type=int, help='Minimum read length to keep') parser.add_argument('-l', '--trim_len', default=300, type=int, help='Trim length for reads') parser.add_argument('-p', '--pad', default='off', choices=['on', 'off'], help='Pad with Ns to a set length') parser.add_argument('--cpus', type=int, help="Number of CPUs. Default: auto") parser.add_argument('-u', '--usearch', dest="usearch", default='usearch9', help='USEARCH9 EXE') parser.add_argument('--cleanup', action='store_true', help='remove intermediate files') parser.add_argument('--merge_method', default='usearch', choices=['usearch', 'vsearch'], help='Software to use for PE read merging') args = parser.parse_args(args) args.out = re.sub(r'\W+', '', args.out) log_name = args.out + '.amptk-demux.log' if os.path.isfile(log_name): os.remove(log_name) amptklib.setupLogging(log_name) FNULL = open(os.devnull, 'w') cmd_args = " ".join(sys.argv) + '\n' amptklib.log.debug(cmd_args) print("-------------------------------------------------------") #initialize script, log system info and usearch version amptklib.SystemInfo() #get version of amptk usearch = args.usearch amptklib.versionDependencyChecks(usearch) #get number of CPUs to use if not args.cpus: cpus = multiprocessing.cpu_count() else: cpus = args.cpus #create tmpdir tmpdir = args.out.split('.')[0] + '_' + str(os.getpid()) if not os.path.exists(tmpdir): os.makedirs(tmpdir) #parse a mapping file or a barcode fasta file, primers, etc get setup #dealing with Barcodes, get ion barcodes or parse the barcode_fasta argument barcode_file = args.out + ".barcodes_used.fa" if os.path.isfile(barcode_file): os.remove(barcode_file) #check if mapping file passed, use this if present, otherwise use command line arguments SampleData = {} Barcodes = {} RevBarcodes = {} FwdPrimer = '' RevPrimer = '' if args.mapping_file: if not os.path.isfile(args.mapping_file): amptklib.log.error("Mapping file not found: %s" % args.mapping_file) sys.exit(1) SampleData, Barcodes, RevBarcodes, FwdPrimer, RevPrimer = amptklib.parseMappingFileNEW( args.mapping_file) else: #no mapping file, so create dictionaries from barcode fasta files if not args.barcode_fasta: amptklib.log.error( "You did not specify a --barcode_fasta or --mapping_file, one is required" ) sys.exit(1) else: shutil.copyfile(args.barcode_fasta, barcode_file) Barcodes = amptklib.fasta2barcodes(barcode_file, False) if FwdPrimer == '' or RevPrimer == '': #parse primers here so doesn't conflict with mapping primers #look up primer db otherwise default to entry if args.F_primer in amptklib.primer_db: FwdPrimer = amptklib.primer_db.get(args.F_primer) amptklib.log.info( "{:} fwd primer found in AMPtk primer db, setting to: {:}". format(args.F_primer, FwdPrimer)) else: FwdPrimer = args.F_primer amptklib.log.info( "{:} fwd primer not found in AMPtk primer db, assuming it is actual primer sequence." .format(args.F_primer)) if args.R_primer in amptklib.primer_db: RevPrimer = amptklib.primer_db.get(args.R_primer) amptklib.log.info( "{:} rev primer found in AMPtk primer db, setting to: {:}". format(args.R_primer, RevPrimer)) else: RevPrimer = args.R_primer amptklib.log.info( "{:} rev primer not found in AMPtk primer db, assuming it is actual primer sequence." .format(args.R_primer)) #if still no primers set, then exit if FwdPrimer == '' or RevPrimer == '': amptklib.log.error( "Please provide primer sequences via --fwd_primer and --rev_primer" ) sys.exit(1) #if barcodes_rev_comp passed then reverse complement the keys in mapdict if args.barcode_rev_comp: amptklib.log.info("Reverse complementing barcode sequences") backupDict = Barcodes Barcodes = {} for k, v in list(backupDict.items()): RCkey = amptklib.RevComp(v) Barcodes[k] = RCkey amptklib.log.info("Loading %i samples from mapping file" % len(Barcodes)) amptklib.log.info('FwdPrimer: {:} RevPrimer: {:}'.format( FwdPrimer, RevPrimer)) amptklib.log.info( 'Dropping reads less than {:} bp and setting lossless trimming to {:} bp.' .format(args.min_len, args.trim_len)) #rename reads according to indexes if not amptklib.PEandIndexCheck( args.fastq, args.reverse, args.index[0]): #check they are all same length amptklib.log.error("FASTQ input malformed, read numbers do not match") sys.exit(1) amptklib.log.info("Loading FASTQ Records") NumSeqs = amptklib.countfastq(args.fastq) if cpus > 1: amptklib.log.info("Splitting FASTQ files over {:} cpus".format(cpus)) amptklib.split_fastqPEandI(args.fastq, args.reverse, args.index[0], NumSeqs, tmpdir, cpus * 2) file_list = [] for file in os.listdir(tmpdir): if file.endswith('.fq'): filepart = os.path.join(tmpdir, file.split('_R')[0]) if not filepart in file_list: file_list.append(filepart) amptklib.log.info("Mapping indexes to reads and renaming PE reads") amptklib.runMultiProgress(safe_run, file_list, cpus, args=args) else: amptklib.log.info("Mapping indexes to reads and renaming PE reads") shutil.copyfile(args.fastq, os.path.join(tmpdir, 'chunk_R1.fq')) shutil.copyfile(args.reverse, os.path.join(tmpdir, 'chunk_R2.fq')) shutil.copyfile(args.index[0], os.path.join(tmpdir, 'chunk_R3.fq')) processReadsPE(os.path.join(tmpdir, 'chunk'), args=args) print("-------------------------------------------------------") #Now concatenate all of the demuxed files together amptklib.log.info("Concatenating Demuxed Files") tmpDemux = os.path.join(tmpdir, args.out + '.demux.fq') with open(tmpDemux, 'wb') as outfile: for filename in glob.glob(os.path.join(tmpdir, '*.demux.fq')): if filename == tmpDemux: continue with open(filename, 'r') as readfile: shutil.copyfileobj(readfile, outfile) #parse the stats finalstats = [0, 0, 0, 0, 0, 0] for file in os.listdir(tmpdir): if file.endswith('.stats'): with open(os.path.join(tmpdir, file), 'r') as statsfile: line = statsfile.readline() line = line.replace('\n', '') newstats = line.split(',') newstats = [int(i) for i in newstats] for x, num in enumerate(newstats): finalstats[x] += num #finally reindex output #last thing is to re-number of reads as it is possible they could have same name from multitprocessor split Demux = args.out + '.demux.fq' amptklib.fastqreindex(tmpDemux, Demux) amptklib.SafeRemove(tmpDemux) #output stats of the run amptklib.log.info('{0:,}'.format(finalstats[0]) + ' total reads') amptklib.log.info('{0:,}'.format(finalstats[0] - finalstats[1]) + ' discarded no index match') amptklib.log.info('{0:,}'.format(finalstats[2]) + ' Fwd Primer found, {0:,}'.format(finalstats[3]) + ' Rev Primer found') amptklib.log.info('{0:,}'.format(finalstats[4]) + ' discarded too short (< %i bp)' % args.min_len) amptklib.log.info('{0:,}'.format(finalstats[5]) + ' valid output reads') #now loop through data and find barcoded samples, counting each..... BarcodeCount = {} with open(Demux, 'r') as input: header = itertools.islice(input, 0, None, 4) for line in header: ID = line.split("=", 1)[-1].split(";")[0] if ID not in BarcodeCount: BarcodeCount[ID] = 1 else: BarcodeCount[ID] += 1 #now let's count the barcodes found and count the number of times they are found. barcode_counts = "%30s: %s" % ('Sample', 'Count') for k, v in natsorted(list(BarcodeCount.items()), key=lambda k_v: k_v[1], reverse=True): barcode_counts += "\n%30s: %s" % (k, str(BarcodeCount[k])) amptklib.log.info("Found %i barcoded samples\n%s" % (len(BarcodeCount), barcode_counts)) #create mapping file if one doesn't exist genericmapfile = args.out + '.mapping_file.txt' amptklib.CreateGenericMappingFile(Barcodes, {}, FwdPrimer, RevPrimer, genericmapfile, BarcodeCount) #compress the output to save space FinalDemux = Demux + '.gz' amptklib.Fzip(Demux, FinalDemux, cpus) amptklib.removefile(Demux) if args.cleanup: amptklib.SafeRemove(tmpdir) #get file size filesize = os.path.getsize(FinalDemux) readablesize = amptklib.convertSize(filesize) amptklib.log.info("Output file: %s (%s)" % (FinalDemux, readablesize)) amptklib.log.info("Mapping file: %s" % genericmapfile) print("-------------------------------------------------------") if 'darwin' in sys.platform: print(col.WARN + "\nExample of next cmd: " + col.END + "amptk cluster -i %s -o out\n" % (FinalDemux)) else: print("\nExample of next cmd: amptk cluster -i %s -o out\n" % (FinalDemux))
def main(args): global FwdPrimer, RevPrimer, Barcodes, tmpdir parser = argparse.ArgumentParser( prog='amptk-process_ion.py', usage="%(prog)s [options] -i file.fastq\n%(prog)s -h for help menu", description= '''Script finds barcodes, strips forward and reverse primers, relabels, and then trim/pads reads to a set length''', epilog="""Written by Jon Palmer (2015) [email protected]""", formatter_class=MyFormatter) parser.add_argument('-i', '--fastq', '--sff', '--fasta', '--bam', dest='fastq', required=True, help='BAM/FASTQ/SFF/FASTA file') parser.add_argument('-q', '--qual', help='QUAL file (if -i is FASTA)') parser.add_argument('-o', '--out', dest="out", default='ion', help='Base name for output') parser.add_argument('-f', '--fwd_primer', dest="F_primer", default='fITS7-ion', help='Forward Primer') parser.add_argument('-r', '--rev_primer', dest="R_primer", default='ITS4', help='Reverse Primer') parser.add_argument( '-m', '--mapping_file', help='Mapping file: QIIME format can have extra meta data columns') parser.add_argument('-p', '--pad', default='off', choices=['on', 'off'], help='Pad with Ns to a set length') parser.add_argument('--primer_mismatch', default=2, type=int, help='Number of mis-matches in primer') parser.add_argument('--barcode_mismatch', default=0, type=int, help='Number of mis-matches in barcode') parser.add_argument( '--barcode_fasta', default='ionxpress', help='FASTA file containing Barcodes (Names & Sequences)') parser.add_argument('--reverse_barcode', help='FASTA file containing 3 prime Barocdes') parser.add_argument('-b', '--list_barcodes', dest="barcodes", default='all', help='Enter Barcodes used separated by commas') parser.add_argument('--min_len', default=100, type=int, help='Minimum read length to keep') parser.add_argument('-l', '--trim_len', default=300, type=int, help='Trim length for reads') parser.add_argument( '--full_length', action='store_true', help='Keep only full length reads (no trimming/padding)') parser.add_argument('--mult_samples', dest="multi", default='False', help='Combine multiple samples (i.e. FACE1)') parser.add_argument('--ion', action='store_true', help='Input data is Ion Torrent') parser.add_argument('--454', action='store_true', help='Input data is 454') parser.add_argument('--cpus', type=int, help="Number of CPUs. Default: auto") parser.add_argument('-u', '--usearch', dest="usearch", default='usearch9', help='USEARCH EXE') args = parser.parse_args(args) args.out = re.sub(r'\W+', '', args.out) log_name = args.out + '.amptk-demux.log' if os.path.isfile(log_name): os.remove(log_name) FNULL = open(os.devnull, 'w') amptklib.setupLogging(log_name) cmd_args = " ".join(sys.argv) + '\n' amptklib.log.debug(cmd_args) print("-------------------------------------------------------") #initialize script, log system info and usearch version amptklib.SystemInfo() #Do a version check usearch = args.usearch amptklib.versionDependencyChecks(usearch) #get number of CPUs to use if not args.cpus: cpus = multiprocessing.cpu_count() else: cpus = args.cpus #parse a mapping file or a barcode fasta file, primers, etc get setup #dealing with Barcodes, get ion barcodes or parse the barcode_fasta argument barcode_file = args.out + ".barcodes_used.fa" rev_barcode_file = args.out + '.revbarcodes_used.fa' amptklib.SafeRemove(barcode_file) amptklib.SafeRemove(rev_barcode_file) #check if mapping file passed, use this if present, otherwise use command line arguments SampleData = {} Barcodes = {} RevBarcodes = {} if args.mapping_file: if not os.path.isfile(args.mapping_file): amptklib.log.error("Mapping file not found: %s" % args.mapping_file) sys.exit(1) SampleData, Barcodes, RevBarcodes, FwdPrimer, RevPrimer = amptklib.parseMappingFileNEW( args.mapping_file) genericmapfile = args.mapping_file else: #no mapping file, so create dictionaries from barcode fasta files if args.barcode_fasta == 'ionxpress': #get script path and barcode file name pgm_barcodes = os.path.join(os.path.dirname(amptklib.__file__), 'DB', 'ionxpress_barcodes.fa') elif args.barcode_fasta == 'ioncode': pgm_barcodes = os.path.join(os.path.dirname(amptklib.__file__), 'DB', 'ioncode_barcodes.fa') if args.barcode_fasta == 'ionxpress' or args.barcode_fasta == 'ioncode': if args.barcodes == "all": if args.multi == 'False': shutil.copyfile(pgm_barcodes, barcode_file) else: with open(barcode_file, 'w') as barcodeout: with open(pgm_barcodes, 'r') as input: for rec in SeqIO.parse(input, 'fasta'): outname = args.multi + '.' + rec.id barcodeout.write(">%s\n%s\n" % (outname, rec.seq)) else: bc_list = args.barcodes.split(",") inputSeqFile = open(pgm_barcodes, "rU") SeqRecords = SeqIO.to_dict(SeqIO.parse(inputSeqFile, "fasta")) for rec in bc_list: name = "BC." + rec seq = SeqRecords[name].seq if args.multi != 'False': outname = args.multi + '.' + name else: outname = name outputSeqFile = open(barcode_file, "a") outputSeqFile.write(">%s\n%s\n" % (outname, seq)) outputSeqFile.close() inputSeqFile.close() else: #check for multi_samples and add if necessary if args.multi == 'False': shutil.copyfile(args.barcode_fasta, barcode_file) if args.reverse_barcode: shutil.copyfile(args.reverse_barcode, rev_barcode_file) else: with open(barcode_file, 'w') as barcodeout: with open(args.barcode_fasta, 'r') as input: for rec in SeqIO.parse(input, 'fasta'): outname = args.multi + '.' + rec.id barcodeout.write(">%s\n%s\n" % (outname, rec.seq)) if args.reverse_barcode: with open(rev_barcode_file, 'w') as barcodeout: with open(args.reverse_barcode, 'r') as input: for rec in SeqIO.parse(input, 'fasta'): outname = args.multi + '.' + rec.id barcodeout.write(">%s\n%s\n" % (outname, rec.seq)) #parse primers here so doesn't conflict with mapping primers #look up primer db otherwise default to entry if args.F_primer in amptklib.primer_db: FwdPrimer = amptklib.primer_db.get(args.F_primer) amptklib.log.info( "{:} fwd primer found in AMPtk primer db, setting to: {:}". format(args.F_primer, FwdPrimer)) else: FwdPrimer = args.F_primer amptklib.log.info( "{:} fwd primer not found in AMPtk primer db, assuming it is actual primer sequence." .format(args.F_primer)) if args.R_primer in amptklib.primer_db: RevPrimer = amptklib.primer_db.get(args.R_primer) amptklib.log.info( "{:} rev primer found in AMPtk primer db, setting to: {:}". format(args.R_primer, RevPrimer)) else: RevPrimer = args.R_primer amptklib.log.info( "{:} rev primer not found in AMPtk primer db, assuming it is actual primer sequence." .format(args.R_primer)) #check if input is compressed gzip_list = [] if args.fastq.endswith('.gz'): gzip_list.append(os.path.abspath(args.fastq)) if gzip_list: amptklib.log.info("Gzipped input files detected, uncompressing") for file in gzip_list: file_out = file.replace('.gz', '') amptklib.Funzip(file, file_out, cpus) args.fastq = args.fastq.replace('.gz', '') #if SFF file passed, convert to FASTQ with biopython if args.fastq.endswith(".sff"): if args.barcode_fasta == 'ionxpress': if not args.mapping_file: amptklib.log.error( "You did not specify a --barcode_fasta or --mapping_file, one is required for 454 data" ) sys.exit(1) amptklib.log.info("SFF input detected, converting to FASTQ") SeqIn = args.out + '.sff.extract.fastq' SeqIO.convert(args.fastq, "sff-trim", SeqIn, "fastq") elif args.fastq.endswith(".fas") or args.fastq.endswith( ".fasta") or args.fastq.endswith(".fa"): if not args.qual: amptklib.log.error( "FASTA input detected, however no QUAL file was given. You must have FASTA + QUAL files" ) sys.exit(1) else: if args.barcode_fasta == 'ionxpress': if not args.mapping_file: amptklib.log.error( "You did not specify a --barcode_fasta or --mapping_file, one is required for 454 data" ) sys.exit(1) SeqIn = args.out + '.fastq' amptklib.log.info("FASTA + QUAL detected, converting to FASTQ") amptklib.faqual2fastq(args.fastq, args.qual, SeqIn) elif args.fastq.endswith('.bam'): #so we can convert natively with pybam, however it is 10X slower than bedtools/samtools #since samtools is fastest, lets use that if exists, if not then bedtools, else default to pybam amptklib.log.info("Converting Ion Torrent BAM file to FASTQ") SeqIn = args.out + '.fastq' if amptklib.which('samtools'): cmd = ['samtools', 'fastq', '-@', str(cpus), args.fastq] amptklib.runSubprocess2(cmd, amptklib.log, SeqIn) else: if amptklib.which('bedtools'): cmd = [ 'bedtools', 'bamtofastq', '-i', args.fastq, '-fq', SeqIn ] amptklib.runSubprocess(cmd, amptklib.log) else: #default to pybam amptklib.bam2fastq(args.fastq, SeqIn) else: SeqIn = args.fastq #start here to process the reads, first reverse complement the reverse primer catDemux = args.out + '.demux.fq' origRevPrimer = RevPrimer RevPrimer = amptklib.RevComp(RevPrimer) amptklib.log.info("Foward primer: %s, Rev comp'd rev primer: %s" % (FwdPrimer, RevPrimer)) #then setup barcode dictionary if len(Barcodes) < 1: Barcodes = amptklib.fasta2barcodes(barcode_file, False) #setup for looking for reverse barcode if len(RevBarcodes) < 1 and args.reverse_barcode: if not os.path.isfile(args.reverse_barcode): amptklib.log.info("Reverse barcode is not a valid file, exiting") sys.exit(1) shutil.copyfile(args.reverse_barcode, rev_barcode_file) RevBarcodes = amptklib.fasta2barcodes(rev_barcode_file, True) #Count FASTQ records amptklib.log.info("Loading FASTQ Records") orig_total = amptklib.countfastq(SeqIn) size = amptklib.checkfastqsize(SeqIn) readablesize = amptklib.convertSize(size) amptklib.log.info('{0:,}'.format(orig_total) + ' reads (' + readablesize + ')') #create tmpdir and split input into n cpus tmpdir = args.out.split('.')[0] + '_' + str(os.getpid()) if not os.path.exists(tmpdir): os.makedirs(tmpdir) amptklib.log.info( 'Dropping reads less than {:} bp and setting lossless trimming to {:} bp.' .format(args.min_len, args.trim_len)) if cpus > 1: #split fastq file amptklib.log.info("Splitting FASTQ files over {:} cpus".format(cpus)) amptklib.split_fastq(SeqIn, orig_total, tmpdir, cpus * 2) #now get file list from tmp folder file_list = [] for file in os.listdir(tmpdir): if file.endswith(".fq"): file = os.path.join(tmpdir, file) file_list.append(file) #finally process reads over number of cpus amptklib.runMultiProgress(processRead, file_list, cpus, args=args) else: shutil.copyfile(SeqIn, os.path.join(tmpdir, 'chunk.fq')) processRead(os.path.join(tmpdir, 'chunk.fq'), args=args) print("-------------------------------------------------------") #Now concatenate all of the demuxed files together amptklib.log.info("Concatenating Demuxed Files") tmpDemux = args.out + '.tmp.demux.fq' with open(tmpDemux, 'w') as outfile: for filename in glob.glob(os.path.join(tmpdir, '*.demux.fq')): if filename == tmpDemux: continue with open(filename, 'r') as readfile: shutil.copyfileobj(readfile, outfile) #parse the stats finalstats = [0, 0, 0, 0, 0, 0, 0] for file in os.listdir(tmpdir): if file.endswith('.stats'): with open(os.path.join(tmpdir, file), 'r') as statsfile: line = statsfile.readline() line = line.rstrip() newstats = line.split(',') newstats = [int(i) for i in newstats] for x, num in enumerate(newstats): finalstats[x] += num #clean up tmp folder shutil.rmtree(tmpdir) #last thing is to re-number of reads as it is possible they could have same name from multitprocessor split amptklib.fastqreindex(tmpDemux, catDemux) os.remove(tmpDemux) amptklib.log.info('{0:,}'.format(finalstats[0]) + ' total reads') if args.reverse_barcode: amptklib.log.info('{0:,}'.format(finalstats[0] - finalstats[1] - finalstats[2] - finalstats[4]) + ' valid Fwd and Rev Barcodes') else: amptklib.log.info('{0:,}'.format(finalstats[0] - finalstats[1]) + ' valid Barcode') amptklib.log.info('{0:,}'.format(finalstats[0] - finalstats[1] - finalstats[2]) + ' Fwd Primer found, {0:,}'.format(finalstats[3]) + ' Rev Primer found') amptklib.log.info('{0:,}'.format(finalstats[5]) + ' discarded too short (< %i bp)' % args.min_len) amptklib.log.info('{0:,}'.format(finalstats[6]) + ' valid output reads') #now loop through data and find barcoded samples, counting each..... BarcodeCount = {} with open(catDemux, 'r') as input: header = itertools.islice(input, 0, None, 4) for line in header: ID = line.split("=", 1)[-1].split(";")[0] if ID not in BarcodeCount: BarcodeCount[ID] = 1 else: BarcodeCount[ID] += 1 #now let's count the barcodes found and count the number of times they are found. barcode_counts = "%22s: %s" % ('Sample', 'Count') for k, v in natsorted(list(BarcodeCount.items()), key=lambda k_v: k_v[1], reverse=True): barcode_counts += "\n%22s: %s" % (k, str(BarcodeCount[k])) amptklib.log.info("Found %i barcoded samples\n%s" % (len(BarcodeCount), barcode_counts)) #create a generic mappingfile for downstream processes genericmapfile = args.out + '.mapping_file.txt' if not args.mapping_file: amptklib.CreateGenericMappingFile(Barcodes, RevBarcodes, FwdPrimer, origRevPrimer, genericmapfile, BarcodeCount) else: amptklib.updateMappingFile(args.mapping_file, BarcodeCount, genericmapfile) #compress the output to save space FinalDemux = catDemux + '.gz' amptklib.Fzip(catDemux, FinalDemux, cpus) amptklib.removefile(catDemux) if gzip_list: for file in gzip_list: file = file.replace('.gz', '') amptklib.removefile(file) #get file size filesize = os.path.getsize(FinalDemux) readablesize = amptklib.convertSize(filesize) amptklib.log.info("Output file: %s (%s)" % (FinalDemux, readablesize)) amptklib.log.info("Mapping file: %s" % genericmapfile) print("-------------------------------------------------------") if 'darwin' in sys.platform: print(col.WARN + "\nExample of next cmd: " + col.END + "amptk cluster -i %s -o out\n" % (FinalDemux)) else: print("\nExample of next cmd: amptk cluster -i %s -o out\n" % (FinalDemux))