def main(args): global FwdPrimer, RevPrimer, SampleData, Barcodes, RevBarcodes, tmpdir, usearch parser=argparse.ArgumentParser(prog='amptk-process_ion.py', usage="%(prog)s [options] -i file.fastq\n%(prog)s -h for help menu", description='''Script finds barcodes, strips forward and reverse primers, relabels, and then trim/pads reads to a set length''', epilog="""Written by Jon Palmer (2015) [email protected]""", formatter_class=MyFormatter) parser.add_argument('-i','--fastq', dest='fastq', required=True, help='FASTQ R1 file') parser.add_argument('--reverse', help='Illumina R2 reverse reads') parser.add_argument('-o','--out', dest="out", default='illumina2', help='Base name for output') parser.add_argument('-f','--fwd_primer', dest="F_primer", default='fITS7', help='Forward Primer') parser.add_argument('-r','--rev_primer', dest="R_primer", default='ITS4', help='Reverse Primer') parser.add_argument('-m','--mapping_file', help='Mapping file: QIIME format can have extra meta data columns') parser.add_argument('-p','--pad', default='off', choices=['on', 'off'], help='Pad with Ns to a set length') parser.add_argument('--primer_mismatch', default=2, type=int, help='Number of mis-matches in primer') parser.add_argument('--barcode_mismatch', default=0, type=int, help='Number of mis-matches in barcode') parser.add_argument('--barcode_fasta', help='FASTA file containing Barcodes (Names & Sequences)') parser.add_argument('--barcode_not_anchored', action='store_true', help='Barcodes (indexes) are not at start of reads') parser.add_argument('--reverse_barcode', help='FASTA file containing 3 prime Barocdes') parser.add_argument('--min_len', default=100, type=int, help='Minimum read length to keep') parser.add_argument('-l','--trim_len', default=300, type=int, help='Trim length for reads') parser.add_argument('--full_length', action='store_true', help='Keep only full length reads (no trimming/padding)') parser.add_argument('--merge_method', default='usearch', choices=['usearch', 'vsearch'], help='Software to use for PE read merging') parser.add_argument('--cpus', type=int, help="Number of CPUs. Default: auto") parser.add_argument('-u','--usearch', dest="usearch", default='usearch9', help='USEARCH EXE') args=parser.parse_args(args) args.out = re.sub(r'\W+', '', args.out) log_name = args.out + '.amptk-demux.log' if os.path.isfile(log_name): os.remove(log_name) FNULL = open(os.devnull, 'w') amptklib.setupLogging(log_name) cmd_args = " ".join(sys.argv)+'\n' amptklib.log.debug(cmd_args) print("-------------------------------------------------------") #initialize script, log system info and usearch version amptklib.SystemInfo() #Do a version check usearch = args.usearch amptklib.versionDependencyChecks(usearch) #get number of CPUs to use if not args.cpus: cpus = multiprocessing.cpu_count() else: cpus = args.cpus #parse a mapping file or a barcode fasta file, primers, etc get setup #dealing with Barcodes, get ion barcodes or parse the barcode_fasta argument barcode_file = args.out + ".barcodes_used.fa" rev_barcode_file = args.out + '.revbarcodes_used.fa' amptklib.SafeRemove(barcode_file) amptklib.SafeRemove(rev_barcode_file) #check if mapping file passed, use this if present, otherwise use command line arguments SampleData = {} Barcodes = {} RevBarcodes = {} FwdPrimer = '' RevPrimer = '' if args.mapping_file: if not os.path.isfile(args.mapping_file): amptklib.log.error("Mapping file not found: %s" % args.mapping_file) sys.exit(1) SampleData, Barcodes, RevBarcodes, FwdPrimer, RevPrimer = amptklib.parseMappingFileNEW(args.mapping_file) else: #no mapping file, so create dictionaries from barcode fasta files if not args.barcode_fasta: amptklib.log.error("You did not specify a --barcode_fasta or --mapping_file, one is required") sys.exit(1) else: shutil.copyfile(args.barcode_fasta, barcode_file) Barcodes = amptklib.fasta2barcodes(barcode_file, False) if args.reverse_barcode: shutil.copyfile(args.reverse_barcode, rev_barcode_file) RevBarcodes = amptklib.fasta2barcodes(rev_barcode_file, False) #parse primers here so doesn't conflict with mapping primers #look up primer db otherwise default to entry if args.F_primer in amptklib.primer_db: FwdPrimer = amptklib.primer_db.get(args.F_primer) amptklib.log.info("{:} fwd primer found in AMPtk primer db, setting to: {:}".format(args.F_primer, FwdPrimer)) else: FwdPrimer = args.F_primer amptklib.log.info("{:} fwd primer not found in AMPtk primer db, assuming it is actual primer sequence.".format(args.F_primer)) if args.R_primer in amptklib.primer_db: RevPrimer = amptklib.primer_db.get(args.R_primer) amptklib.log.info("{:} rev primer found in AMPtk primer db, setting to: {:}".format(args.R_primer, RevPrimer)) else: RevPrimer = args.R_primer amptklib.log.info("{:} rev primer not found in AMPtk primer db, assuming it is actual primer sequence.".format(args.R_primer)) #check if input is compressed gzip_list = [] if args.fastq.endswith('.gz'): gzip_list.append(os.path.abspath(args.fastq)) if args.reverse: if args.reverse.endswith('.gz'): gzip_list.append(os.path.abspath(args.reverse)) if gzip_list: amptklib.log.info("Gzipped input files detected, uncompressing") for file in gzip_list: file_out = file.replace('.gz', '') amptklib.Funzip(file, file_out, cpus) args.fastq = args.fastq.replace('.gz', '') if args.reverse: args.reverse = args.reverse.replace('.gz', '') #Count FASTQ records amptklib.log.info("Loading FASTQ Records") orig_total = amptklib.countfastq(args.fastq) size = amptklib.checkfastqsize(args.fastq) readablesize = amptklib.convertSize(size*2) amptklib.log.info('{:,} reads ({:})'.format(orig_total, readablesize)) #output barcodes/samples amptklib.log.info('Searching for {:} forward barcodes and {:} reverse barcodes'.format(len(Barcodes), len(RevBarcodes))) #create tmpdir and split input into n cpus tmpdir = args.out.split('.')[0]+'_'+str(os.getpid()) if not os.path.exists(tmpdir): os.makedirs(tmpdir) #tell user about number of cores using amptklib.log.info("Splitting FASTQ files over {:} cpus".format(cpus)) if args.reverse: amptklib.log.info("Demuxing PE Illumina reads; FwdPrimer: {:} RevPrimer: {:}".format(FwdPrimer, RevPrimer)) else: amptklib.log.info("Demuxing SE Illumina reads; FwdPrimer: {:} RevPrimer: {:}".format(FwdPrimer, amptklib.RevComp(RevPrimer))) amptklib.log.info('Dropping reads less than {:} bp and setting lossless trimming to {:} bp.'.format(args.min_len, args.trim_len)) if cpus > 1: if args.reverse: amptklib.split_fastqPE(args.fastq, args.reverse, orig_total, tmpdir, cpus*4) file_list = [] for file in os.listdir(tmpdir): if file.endswith('.fq'): filepart = os.path.join(tmpdir, file.split('_R')[0]) if not filepart in file_list: file_list.append(filepart) amptklib.runMultiProgress(processReadsPE, file_list, cpus, args=args) else: #split fastq file amptklib.split_fastq(args.fastq, orig_total, tmpdir, cpus*4) #now get file list from tmp folder file_list = [] for file in os.listdir(tmpdir): if file.endswith(".fq"): file = os.path.join(tmpdir, file) file_list.append(file) #finally process reads over number of cpus amptklib.runMultiProgress(processRead, file_list, cpus, args=args) else: if args.reverse: shutil.copyfile(args.fastq, os.path.join(tmpdir, 'chunk_R1.fq')) shutil.copyfile(args.reverse, os.path.join(tmpdir, 'chunk_R2.fq')) processReadsPE(os.path.join(tmpdir, 'chunk'), args=args) else: shutil.copyfile(args.fastq, os.path.join(tmpdir, 'chunk.fq')) processRead(os.path.join(tmpdir, 'chunk.fq'), args=args) print("-------------------------------------------------------") #Now concatenate all of the demuxed files together amptklib.log.info("Concatenating Demuxed Files") tmpDemux = args.out + '.tmp.demux.fq' with open(tmpDemux, 'w') as outfile: for filename in glob.glob(os.path.join(tmpdir,'*.demux.fq')): if filename == tmpDemux: continue with open(filename, 'r') as readfile: shutil.copyfileobj(readfile, outfile) if args.reverse: #parse the stats finalstats = [0,0,0,0,0,0] for file in os.listdir(tmpdir): if file.endswith('.stats'): with open(os.path.join(tmpdir, file), 'r') as statsfile: line = statsfile.readline() line = line.rstrip() newstats = line.split(',') newstats = [int(i) for i in newstats] for x, num in enumerate(newstats): finalstats[x] += num amptklib.log.info('{0:,}'.format(finalstats[0])+' total reads') amptklib.log.info('{0:,}'.format(finalstats[0]-finalstats[1]-finalstats[3])+' valid Barcodes') amptklib.log.info('{0:,}'.format(finalstats[5])+' valid output reads (Barcodes and Primers)') else: #parse the stats finalstats = [0,0,0,0,0,0,0] for file in os.listdir(tmpdir): if file.endswith('.stats'): with open(os.path.join(tmpdir, file), 'r') as statsfile: line = statsfile.readline() line = line.rstrip() newstats = line.split(',') newstats = [int(i) for i in newstats] for x, num in enumerate(newstats): finalstats[x] += num amptklib.log.info('{0:,}'.format(finalstats[0])+' total reads') if args.reverse_barcode: amptklib.log.info('{0:,}'.format(finalstats[0]-finalstats[1]-finalstats[2]-finalstats[4])+' valid Fwd and Rev Barcodes') else: amptklib.log.info('{0:,}'.format(finalstats[0]-finalstats[1])+' valid Barcode') amptklib.log.info('{0:,}'.format(finalstats[0]-finalstats[1]-finalstats[2])+' Fwd Primer found, {0:,}'.format(finalstats[3])+ ' Rev Primer found') amptklib.log.info('{0:,}'.format(finalstats[5])+' discarded too short (< %i bp)' % args.min_len) amptklib.log.info('{0:,}'.format(finalstats[6])+' valid output reads') #clean up tmp folder amptklib.SafeRemove(tmpdir) #last thing is to re-number of reads as it is possible they could have same name from multitprocessor split catDemux = args.out+'.demux.fq' amptklib.fastqreindex(tmpDemux, catDemux) amptklib.SafeRemove(tmpDemux) #now loop through data and find barcoded samples, counting each..... BarcodeCount = {} with open(catDemux, 'r') as input: header = itertools.islice(input, 0, None, 4) for line in header: ID = line.split("=",1)[-1].split(";")[0] if ID not in BarcodeCount: BarcodeCount[ID] = 1 else: BarcodeCount[ID] += 1 #now let's count the barcodes found and count the number of times they are found. barcode_counts = "%22s: %s" % ('Sample', 'Count') for k,v in natsorted(list(BarcodeCount.items()), key=lambda k_v: k_v[1], reverse=True): barcode_counts += "\n%22s: %s" % (k, str(BarcodeCount[k])) amptklib.log.info("Found %i barcoded samples\n%s" % (len(BarcodeCount), barcode_counts)) genericmapfile = args.out + '.mapping_file.txt' if not args.mapping_file: #create a generic mappingfile for downstream processes amptklib.CreateGenericMappingFile(Barcodes, RevBarcodes, FwdPrimer, RevPrimer, genericmapfile, BarcodeCount) else: amptklib.updateMappingFile(args.mapping_file, BarcodeCount, genericmapfile) #compress the output to save space FinalDemux = catDemux+'.gz' amptklib.Fzip(catDemux, FinalDemux, cpus) amptklib.removefile(catDemux) if gzip_list: for file in gzip_list: file = file.replace('.gz', '') amptklib.removefile(file) #get file size filesize = os.path.getsize(FinalDemux) readablesize = amptklib.convertSize(filesize) amptklib.log.info("Output file: %s (%s)" % (FinalDemux, readablesize)) amptklib.log.info("Mapping file: %s" % genericmapfile) print("-------------------------------------------------------") if 'darwin' in sys.platform: print(col.WARN + "\nExample of next cmd: " + col.END + "amptk cluster -i %s -o out\n" % (FinalDemux)) else: print("\nExample of next cmd: amptk cluster -i %s -o out\n" % (FinalDemux))
def main(args):
global FwdPrimer, RevPrimer, usearch
parser = argparse.ArgumentParser(
prog='amptk-process_illumina_folder.py',
usage="%(prog)s [options] -i folder",
description=
'''Script that takes De-mulitplexed Illumina data from a folder and processes it for amptk (merge PE reads, strip primers, trim/pad to set length.''',
epilog="""Written by Jon Palmer (2015) [email protected]""",
formatter_class=MyFormatter)
parser.add_argument('-i',
'--input',
dest='input',
required=True,
help='Folder of Illumina Data')
parser.add_argument('-o',
'--out',
dest="out",
default='amptk-illumina',
help='Name for output folder')
parser.add_argument(
'-m',
'--mapping_file',
help='Mapping file: QIIME format can have extra meta data columns')
parser.add_argument('--reads',
dest="reads",
default='paired',
choices=['paired', 'forward'],
help='PE or forward reads')
parser.add_argument('--read_length',
type=int,
help='Read length, i.e. 2 x 300 bp = 300')
parser.add_argument('-f',
'--fwd_primer',
dest="F_primer",
default='fITS7',
help='Forward Primer (fITS7)')
parser.add_argument('-r',
'--rev_primer',
dest="R_primer",
default='ITS4',
help='Reverse Primer (ITS4)')
parser.add_argument('--require_primer',
dest="primer",
default='on',
choices=['on', 'off'],
help='Require Fwd primer to be present')
parser.add_argument('--primer_mismatch',
default=2,
type=int,
help='Number of mis-matches in primer')
parser.add_argument('--barcode_mismatch',
default=1,
type=int,
help='Number of mis-matches allowed in index')
parser.add_argument('--rescue_forward',
default='on',
choices=['on', 'off'],
help='Rescue Not-merged forward reads')
parser.add_argument('--min_len',
default=100,
type=int,
help='Minimum read length to keep')
parser.add_argument('--merge_method',
default='usearch',
choices=['usearch', 'vsearch'],
help='Software to use for PE read merging')
parser.add_argument('-l',
'--trim_len',
default=300,
type=int,
help='Trim length for reads')
parser.add_argument('--cpus',
type=int,
help="Number of CPUs. Default: auto")
parser.add_argument(
'--full_length',
action='store_true',
help='Keep only full length reads (no trimming/padding)')
parser.add_argument('-p',
'--pad',
default='off',
choices=['on', 'off'],
help='Pad with Ns to a set length')
parser.add_argument('-u',
'--usearch',
dest="usearch",
default='usearch9',
help='USEARCH executable')
parser.add_argument(
'--sra',
action='store_true',
help='Input files are from NCBI SRA not direct from illumina')
parser.add_argument('--cleanup',
action='store_true',
help='Delete all intermediate files')
args = parser.parse_args(args)
#sometimes people add slashes in the output directory, this could be bad, try to fix it
args.out = re.sub(r'\W+', '', args.out)
#create directory and check for existing logfile
if not os.path.exists(args.out):
os.makedirs(args.out)
log_name = args.out + '.amptk-demux.log'
if os.path.isfile(log_name):
os.remove(log_name)
amptklib.setupLogging(log_name)
FNULL = open(os.devnull, 'w')
cmd_args = " ".join(sys.argv) + '\n'
amptklib.log.debug(cmd_args)
print("-------------------------------------------------------")
#initialize script, log system info and usearch version
amptklib.SystemInfo()
#get version of amptk
usearch = args.usearch
amptklib.versionDependencyChecks(usearch)
#Now all the data is in folder args.out that needs to be de-multiplexed
if not args.cpus:
cpus = multiprocessing.cpu_count()
else:
cpus = args.cpus
#check folder if files are gzipped, then gunzip them
#try to gunzip files
gzip_list = []
for file in os.listdir(args.input):
if file.endswith(".fastq.gz"):
gzip_list.append(file)
if gzip_list:
amptklib.log.info("Gzipped files detected, uncompressing")
for file in gzip_list:
amptklib.log.debug("Uncompressing %s" % file)
OutName = os.path.join(args.input, os.path.splitext(file)[0])
amptklib.Funzip(os.path.join(args.input, file), OutName, cpus)
#check for mapping file, if exists, then use names from first column only for filenames
SampleData = {}
Barcodes = {}
RevBarcodes = {}
FwdPrimer = ''
RevPrimer = ''
if args.mapping_file:
if not os.path.isfile(args.mapping_file):
amptklib.error("Mapping file is not valid: %s" % args.mapping_file)
sys.exit(1)
SampleData, Barcodes, RevBarcodes, FwdPrimer, RevPrimer = amptklib.parseMappingFileNEW(
args.mapping_file)
mapdata = amptklib.parseMappingFileIllumina(args.mapping_file)
#forward primer in first item in tuple, reverse in second
sample_names = list(SampleData.keys())
#loop through the files in the folder and get the ones in the sample_names lit
filenames = []
for file in os.listdir(args.input):
if file.startswith(tuple(sample_names)):
if file.endswith('.fastq'):
filenames.append(file)
if len(filenames) < 1:
amptklib.log.error(
"Found 0 valid files from mapping file. Mapping file SampleID must match start of filenames"
)
sys.exit(1)
else: #if not then search through and find all the files you can in the folder
'''get filenames, store in list, Illumina file names look like the following:
<sample name>_<i5>-<i7>_L<lane (0-padded to 3 digits)>_R<read number>_<set number (0-padded to 3 digits>.fastq.gz'''
#now get the FASTQ files and proceed
filenames = []
for file in os.listdir(args.input):
if file.endswith(".fastq"):
filenames.append(file)
#look up primer db otherwise default to entry
if args.F_primer in amptklib.primer_db:
FwdPrimer = amptklib.primer_db.get(args.F_primer)
amptklib.log.info(
"{:} fwd primer found in AMPtk primer db, setting to: {:}".
format(args.F_primer, FwdPrimer))
else:
FwdPrimer = args.F_primer
amptklib.log.info(
"{:} fwd primer not found in AMPtk primer db, assuming it is actual primer sequence."
.format(args.F_primer))
if args.R_primer in amptklib.primer_db:
RevPrimer = amptklib.primer_db.get(args.R_primer)
amptklib.log.info(
"{:} rev primer found in AMPtk primer db, setting to: {:}".
format(args.R_primer, RevPrimer))
else:
RevPrimer = args.R_primer
amptklib.log.info(
"{:} rev primer not found in AMPtk primer db, assuming it is actual primer sequence."
.format(args.R_primer))
#if files are from SRA, then do something different as they are already merged
if args.sra:
#take list of filenames, move over to output folder
sampleDict = {}
fastq_for = []
for x in filenames:
rename = os.path.basename(x).split(".f", -1)[0]
sampleDict[rename] = 'unknown'
shutil.copyfile(os.path.join(args.input, x),
os.path.join(args.out, rename + '.fq'))
fastq_for.append(os.path.join(args.out, rename + '.fq'))
args.reads = 'forward'
else:
if len(filenames) % 2 != 0:
print(
"Check your input files, they do not seem to be properly paired"
)
sys.exit(1)
#check list for files, i.e. they need to have _R1 and _R2 in the filenames, otherwise throw exception
if not any('_R1' in x for x in filenames):
amptklib.log.error(
"Did not find valid FASTQ files. Your files must have _R1 and _R2 in filename, rename your files and restart script."
)
sys.exit(1)
uniq_names = []
fastq_for = []
fastq_rev = []
sampleDict = {}
map = args.out + '.filenames.txt'
with open(map, 'w') as map_file:
map_file.write("Name\t[i5]\t[i7]\tLane\tSet_num\n")
for item in sorted(filenames):
if '_R1' in item:
fastq_for.append(os.path.join(args.input, item))
if '_R2' in item:
fastq_rev.append(os.path.join(args.input, item))
column = item.split("_")
if column[0] not in uniq_names:
uniq_names.append(column[0])
if "-" in column[1]:
barcode = column[1].split(
"-"
) #looking here for the linker between i5 and i7 seqs
i5 = barcode[0]
i7 = barcode[1]
try:
map_file.write("%s\t%s\t%s\t%s\t%s\n" %
(column[0], i5, i7, column[2],
column[4].split(".", 1)[0]))
except IndexError:
amptklib.log.debug(
"Non-standard names detected, skipping mapping file"
)
else:
i5 = column[1]
i7 = "None"
try:
map_file.write("%s\t%s\t%s\t%s\t%s\n" %
(column[0], i5, i7, column[2],
column[4].split(".", 1)[0]))
except IndexError:
amptklib.log.debug(
"Non-standard names detected, skipping mapping file"
)
if i7 != "None":
sampleDict[column[0]] = i5 + '-' + i7
else:
sampleDict[column[0]] = i5
if args.full_length and args.primer == 'off':
amptklib.log.info(
'--full_length is not compatible with --require_primer off, turning --full_length off'
)
args.full_length = False
#tell user about number of cores using
amptklib.log.info('Demuxing data using {:} cpus'.format(cpus))
amptklib.log.info(
'Dropping reads less than {:} bp and setting lossless trimming to {:} bp.'
.format(args.min_len, args.trim_len))
#zip read lists into a single list of tuples
if args.reads == 'paired':
amptklib.log.info(
"Strip Primers and Merge PE reads. FwdPrimer: {:} RevPrimer: {:}".
format(FwdPrimer, RevPrimer))
readList = list(zip(fastq_for, fastq_rev))
amptklib.runMultiProgress(safe_run, readList, cpus, args=args)
else:
amptklib.log.info(
"Strip Primers. FwdPrimer: {:} RevPrimer: {:}".format(
FwdPrimer, RevPrimer))
amptklib.runMultiProgress(safe_run2, fastq_for, cpus, args=args)
#cleanup to save space
if gzip_list:
for file in gzip_list:
file = file.replace('.gz', '')
amptklib.removefile(os.path.join(args.input, file))
print("-------------------------------------------------------")
#Now concatenate all of the demuxed files together
amptklib.log.info("Concatenating Demuxed Files")
catDemux = args.out + '.demux.fq'
with open(catDemux, 'w') as outfile:
for filename in glob.glob(os.path.join(args.out, '*.demux.fq')):
if filename == catDemux:
continue
with open(filename, 'r') as readfile:
shutil.copyfileobj(readfile, outfile)
#parse the stats
#(Total, ForPrimerFound, RevPrimerFound, multiHits, TooShort, ValidSeqs))
finalstats = [0, 0, 0, 0, 0, 0]
for file in os.listdir(args.out):
if file.endswith('.stats'):
with open(os.path.join(args.out, file), 'r') as statsfile:
line = statsfile.readline()
line = line.replace('\n', '')
newstats = line.split(',')
newstats = [int(i) for i in newstats]
for x, num in enumerate(newstats):
finalstats[x] += num
amptklib.log.info('{0:,}'.format(finalstats[0]) + ' total reads')
amptklib.log.info('{0:,}'.format(finalstats[1]) +
' Fwd Primer found, {0:,}'.format(finalstats[2]) +
' Rev Primer found')
amptklib.log.info('{0:,}'.format(finalstats[3]) +
' discarded Primer incompatibility')
amptklib.log.info('{0:,}'.format(finalstats[4]) +
' discarded too short (< %i bp)' % args.min_len)
amptklib.log.info('{0:,}'.format(finalstats[5]) + ' valid output reads')
#now loop through data and find barcoded samples, counting each.....
BarcodeCount = {}
with open(catDemux, 'r') as input:
header = itertools.islice(input, 0, None, 4)
for line in header:
ID = line.split("=")[-1].split(";")[0]
if ID not in BarcodeCount:
BarcodeCount[ID] = 1
else:
BarcodeCount[ID] += 1
#now let's count the barcodes found and count the number of times they are found.
barcode_counts = "%30s: %s" % ('Sample', 'Count')
for k, v in natsorted(list(BarcodeCount.items()),
key=lambda k_v: k_v[1],
reverse=True):
barcode_counts += "\n%30s: %s" % (k, str(BarcodeCount[k]))
amptklib.log.info("Found %i barcoded samples\n%s" %
(len(BarcodeCount), barcode_counts))
genericmapfile = args.out + '.mapping_file.txt'
if not args.mapping_file:
#create a generic mappingfile for downstream processes
amptklib.CreateGenericMappingFileIllumina(sampleDict, FwdPrimer,
RevPrimer, genericmapfile,
BarcodeCount)
else:
amptklib.updateMappingFile(args.mapping_file, BarcodeCount,
genericmapfile)
#compress the output to save space
FinalDemux = catDemux + '.gz'
amptklib.Fzip(catDemux, FinalDemux, cpus)
amptklib.removefile(catDemux)
#get file size
filesize = os.path.getsize(FinalDemux)
readablesize = amptklib.convertSize(filesize)
amptklib.log.info("Output file: %s (%s)" % (FinalDemux, readablesize))
amptklib.log.info("Mapping file: %s" % genericmapfile)
if args.cleanup:
shutil.rmtree(args.out)
print("-------------------------------------------------------")
if 'darwin' in sys.platform:
print(col.WARN + "\nExample of next cmd: " + col.END +
"amptk cluster -i %s -o out\n" % (FinalDemux))
else:
print("\nExample of next cmd: amptk cluster -i %s -o out\n" %
(FinalDemux))
def main(args):
global FwdPrimer, RevPrimer, Barcodes, tmpdir, usearch
parser = argparse.ArgumentParser(
prog='amptk-process_illumina_raw.py',
usage="%(prog)s [options] -i file.fastq\n%(prog)s -h for help menu",
description=
'''Script finds barcodes, strips forward and reverse primers, relabels, and then trim/pads reads to a set length''',
epilog="""Written by Jon Palmer (2015) [email protected]""",
formatter_class=MyFormatter)
parser.add_argument('-f',
'--forward',
dest='fastq',
required=True,
help='Illumina FASTQ R1 reads')
parser.add_argument('-r',
'--reverse',
required=True,
help='Illumina FASTQ R2 reads')
parser.add_argument('-i',
'--index',
nargs='+',
required=True,
help='Illumina FASTQ index reads')
parser.add_argument('-m', '--mapping_file', help='QIIME-like mapping file')
parser.add_argument('--read_length',
type=int,
help='Read length, i.e. 2 x 300 bp = 300')
parser.add_argument('-o',
'--out',
dest="out",
default='illumina_out',
help='Base name for output')
parser.add_argument('--fwd_primer',
dest="F_primer",
default='515FB',
help='Forward Primer')
parser.add_argument('--rev_primer',
dest="R_primer",
default='806RB',
help='Reverse Primer')
parser.add_argument('--primer_mismatch',
default=2,
type=int,
help='Number of mis-matches in primer')
parser.add_argument('--barcode_mismatch',
default=0,
type=int,
help='Number of mis-matches in barcode')
parser.add_argument(
'--barcode_fasta',
help='FASTA file containing Barcodes (Names & Sequences)')
parser.add_argument('--rescue_forward',
default='on',
choices=['on', 'off'],
help='Rescue Not-merged forward reads')
parser.add_argument('--barcode_rev_comp',
action='store_true',
help='Reverse complement barcode sequences')
parser.add_argument('--min_len',
default=100,
type=int,
help='Minimum read length to keep')
parser.add_argument('-l',
'--trim_len',
default=300,
type=int,
help='Trim length for reads')
parser.add_argument('-p',
'--pad',
default='off',
choices=['on', 'off'],
help='Pad with Ns to a set length')
parser.add_argument('--cpus',
type=int,
help="Number of CPUs. Default: auto")
parser.add_argument('-u',
'--usearch',
dest="usearch",
default='usearch9',
help='USEARCH9 EXE')
parser.add_argument('--cleanup',
action='store_true',
help='remove intermediate files')
parser.add_argument('--merge_method',
default='usearch',
choices=['usearch', 'vsearch'],
help='Software to use for PE read merging')
args = parser.parse_args(args)
args.out = re.sub(r'\W+', '', args.out)
log_name = args.out + '.amptk-demux.log'
if os.path.isfile(log_name):
os.remove(log_name)
amptklib.setupLogging(log_name)
FNULL = open(os.devnull, 'w')
cmd_args = " ".join(sys.argv) + '\n'
amptklib.log.debug(cmd_args)
print("-------------------------------------------------------")
#initialize script, log system info and usearch version
amptklib.SystemInfo()
#get version of amptk
usearch = args.usearch
amptklib.versionDependencyChecks(usearch)
#get number of CPUs to use
if not args.cpus:
cpus = multiprocessing.cpu_count()
else:
cpus = args.cpus
#create tmpdir
tmpdir = args.out.split('.')[0] + '_' + str(os.getpid())
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
#parse a mapping file or a barcode fasta file, primers, etc get setup
#dealing with Barcodes, get ion barcodes or parse the barcode_fasta argument
barcode_file = args.out + ".barcodes_used.fa"
if os.path.isfile(barcode_file):
os.remove(barcode_file)
#check if mapping file passed, use this if present, otherwise use command line arguments
SampleData = {}
Barcodes = {}
RevBarcodes = {}
FwdPrimer = ''
RevPrimer = ''
if args.mapping_file:
if not os.path.isfile(args.mapping_file):
amptklib.log.error("Mapping file not found: %s" %
args.mapping_file)
sys.exit(1)
SampleData, Barcodes, RevBarcodes, FwdPrimer, RevPrimer = amptklib.parseMappingFileNEW(
args.mapping_file)
else: #no mapping file, so create dictionaries from barcode fasta files
if not args.barcode_fasta:
amptklib.log.error(
"You did not specify a --barcode_fasta or --mapping_file, one is required"
)
sys.exit(1)
else:
shutil.copyfile(args.barcode_fasta, barcode_file)
Barcodes = amptklib.fasta2barcodes(barcode_file, False)
if FwdPrimer == '' or RevPrimer == '':
#parse primers here so doesn't conflict with mapping primers
#look up primer db otherwise default to entry
if args.F_primer in amptklib.primer_db:
FwdPrimer = amptklib.primer_db.get(args.F_primer)
amptklib.log.info(
"{:} fwd primer found in AMPtk primer db, setting to: {:}".
format(args.F_primer, FwdPrimer))
else:
FwdPrimer = args.F_primer
amptklib.log.info(
"{:} fwd primer not found in AMPtk primer db, assuming it is actual primer sequence."
.format(args.F_primer))
if args.R_primer in amptklib.primer_db:
RevPrimer = amptklib.primer_db.get(args.R_primer)
amptklib.log.info(
"{:} rev primer found in AMPtk primer db, setting to: {:}".
format(args.R_primer, RevPrimer))
else:
RevPrimer = args.R_primer
amptklib.log.info(
"{:} rev primer not found in AMPtk primer db, assuming it is actual primer sequence."
.format(args.R_primer))
#if still no primers set, then exit
if FwdPrimer == '' or RevPrimer == '':
amptklib.log.error(
"Please provide primer sequences via --fwd_primer and --rev_primer"
)
sys.exit(1)
#if barcodes_rev_comp passed then reverse complement the keys in mapdict
if args.barcode_rev_comp:
amptklib.log.info("Reverse complementing barcode sequences")
backupDict = Barcodes
Barcodes = {}
for k, v in list(backupDict.items()):
RCkey = amptklib.RevComp(v)
Barcodes[k] = RCkey
amptklib.log.info("Loading %i samples from mapping file" % len(Barcodes))
amptklib.log.info('FwdPrimer: {:} RevPrimer: {:}'.format(
FwdPrimer, RevPrimer))
amptklib.log.info(
'Dropping reads less than {:} bp and setting lossless trimming to {:} bp.'
.format(args.min_len, args.trim_len))
#rename reads according to indexes
if not amptklib.PEandIndexCheck(
args.fastq, args.reverse,
args.index[0]): #check they are all same length
amptklib.log.error("FASTQ input malformed, read numbers do not match")
sys.exit(1)
amptklib.log.info("Loading FASTQ Records")
NumSeqs = amptklib.countfastq(args.fastq)
if cpus > 1:
amptklib.log.info("Splitting FASTQ files over {:} cpus".format(cpus))
amptklib.split_fastqPEandI(args.fastq, args.reverse, args.index[0],
NumSeqs, tmpdir, cpus * 2)
file_list = []
for file in os.listdir(tmpdir):
if file.endswith('.fq'):
filepart = os.path.join(tmpdir, file.split('_R')[0])
if not filepart in file_list:
file_list.append(filepart)
amptklib.log.info("Mapping indexes to reads and renaming PE reads")
amptklib.runMultiProgress(safe_run, file_list, cpus, args=args)
else:
amptklib.log.info("Mapping indexes to reads and renaming PE reads")
shutil.copyfile(args.fastq, os.path.join(tmpdir, 'chunk_R1.fq'))
shutil.copyfile(args.reverse, os.path.join(tmpdir, 'chunk_R2.fq'))
shutil.copyfile(args.index[0], os.path.join(tmpdir, 'chunk_R3.fq'))
processReadsPE(os.path.join(tmpdir, 'chunk'), args=args)
print("-------------------------------------------------------")
#Now concatenate all of the demuxed files together
amptklib.log.info("Concatenating Demuxed Files")
tmpDemux = os.path.join(tmpdir, args.out + '.demux.fq')
with open(tmpDemux, 'wb') as outfile:
for filename in glob.glob(os.path.join(tmpdir, '*.demux.fq')):
if filename == tmpDemux:
continue
with open(filename, 'r') as readfile:
shutil.copyfileobj(readfile, outfile)
#parse the stats
finalstats = [0, 0, 0, 0, 0, 0]
for file in os.listdir(tmpdir):
if file.endswith('.stats'):
with open(os.path.join(tmpdir, file), 'r') as statsfile:
line = statsfile.readline()
line = line.replace('\n', '')
newstats = line.split(',')
newstats = [int(i) for i in newstats]
for x, num in enumerate(newstats):
finalstats[x] += num
#finally reindex output
#last thing is to re-number of reads as it is possible they could have same name from multitprocessor split
Demux = args.out + '.demux.fq'
amptklib.fastqreindex(tmpDemux, Demux)
amptklib.SafeRemove(tmpDemux)
#output stats of the run
amptklib.log.info('{0:,}'.format(finalstats[0]) + ' total reads')
amptklib.log.info('{0:,}'.format(finalstats[0] - finalstats[1]) +
' discarded no index match')
amptklib.log.info('{0:,}'.format(finalstats[2]) +
' Fwd Primer found, {0:,}'.format(finalstats[3]) +
' Rev Primer found')
amptklib.log.info('{0:,}'.format(finalstats[4]) +
' discarded too short (< %i bp)' % args.min_len)
amptklib.log.info('{0:,}'.format(finalstats[5]) + ' valid output reads')
#now loop through data and find barcoded samples, counting each.....
BarcodeCount = {}
with open(Demux, 'r') as input:
header = itertools.islice(input, 0, None, 4)
for line in header:
ID = line.split("=", 1)[-1].split(";")[0]
if ID not in BarcodeCount:
BarcodeCount[ID] = 1
else:
BarcodeCount[ID] += 1
#now let's count the barcodes found and count the number of times they are found.
barcode_counts = "%30s: %s" % ('Sample', 'Count')
for k, v in natsorted(list(BarcodeCount.items()),
key=lambda k_v: k_v[1],
reverse=True):
barcode_counts += "\n%30s: %s" % (k, str(BarcodeCount[k]))
amptklib.log.info("Found %i barcoded samples\n%s" %
(len(BarcodeCount), barcode_counts))
#create mapping file if one doesn't exist
genericmapfile = args.out + '.mapping_file.txt'
amptklib.CreateGenericMappingFile(Barcodes, {}, FwdPrimer, RevPrimer,
genericmapfile, BarcodeCount)
#compress the output to save space
FinalDemux = Demux + '.gz'
amptklib.Fzip(Demux, FinalDemux, cpus)
amptklib.removefile(Demux)
if args.cleanup:
amptklib.SafeRemove(tmpdir)
#get file size
filesize = os.path.getsize(FinalDemux)
readablesize = amptklib.convertSize(filesize)
amptklib.log.info("Output file: %s (%s)" % (FinalDemux, readablesize))
amptklib.log.info("Mapping file: %s" % genericmapfile)
print("-------------------------------------------------------")
if 'darwin' in sys.platform:
print(col.WARN + "\nExample of next cmd: " + col.END +
"amptk cluster -i %s -o out\n" % (FinalDemux))
else:
print("\nExample of next cmd: amptk cluster -i %s -o out\n" %
(FinalDemux))
def main(args): parser=argparse.ArgumentParser(prog='amptk-fastq2sra.py', usage="%(prog)s [options] -i folder", description='''Script to split FASTQ file from Ion, 454, or Illumina by barcode sequence into separate files for submission to SRA. This script can take the BioSample worksheet from NCBI and create an SRA metadata file for submission.''', epilog="""Written by Jon Palmer (2015) [email protected]""", formatter_class=MyFormatter) parser.add_argument('-i','--input', dest='FASTQ', required=True, help='Input FASTQ file or folder') parser.add_argument('-o','--out', dest='out', help='Basename for output folder/files') parser.add_argument('--min_len', default=50, type=int, help='Minimum length of read to keep') parser.add_argument('-b','--barcode_fasta', help='Multi-fasta file containing barcodes used') parser.add_argument('--reverse_barcode', help='Reverse barcode fasta file') parser.add_argument('-s','--biosample', dest='biosample', help='BioSample file from NCBI') parser.add_argument('-p','--platform', dest='platform', default='ion', choices=['ion', 'illumina', '454'], help='Sequencing platform') parser.add_argument('-f','--fwd_primer', dest="F_primer", default='fITS7', help='Forward Primer (fITS7)') parser.add_argument('-r','--rev_primer', dest="R_primer", default='ITS4', help='Reverse Primer (ITS4)') parser.add_argument('-n', '--names', help='CSV mapping file BC,NewName') parser.add_argument('-d', '--description', help='Paragraph description for SRA metadata') parser.add_argument('-t','--title', default='Fungal ITS', help='Start of title for SRA submission, name it according to amplicon') parser.add_argument('-m','--mapping_file', help='Mapping file: QIIME format can have extra meta data columns') parser.add_argument('--primer_mismatch', default=2, type=int, help='Number of mis-matches in primer') parser.add_argument('--barcode_mismatch', default=0, type=int, help='Number of mis-matches in barcode') parser.add_argument('--require_primer', default='off', choices=['forward', 'both', 'off'], help='Require Primers to be present') parser.add_argument('--force', action='store_true', help='Overwrite existing directory') parser.add_argument('-a','--append', help='Append a name to all sample names for a run, i.e. --append run1 would yield Sample_run1') args=parser.parse_args(args) #get basename if not args.out passed if args.out: base = args.out else: if 'demux' in args.FASTQ: base = os.path.basename(args.FASTQ).split('.demux')[0] else: base = os.path.basename(args.FASTQ).split('.f')[0] log_name = base + '.amptk-sra.log' if os.path.isfile(log_name): os.remove(log_name) amptklib.setupLogging(log_name) FNULL = open(os.devnull, 'w') cmd_args = " ".join(sys.argv)+'\n' amptklib.log.debug(cmd_args) print("-------------------------------------------------------") amptklib.SystemInfo() amptkversion = amptklib.get_version() #create output directory if not os.path.exists(base): os.makedirs(base) else: if not args.force: amptklib.log.error("Directory %s exists, add --force argument to overwrite" % base) sys.exit(1) else: shutil.rmtree(base) os.makedirs(base) #parse a mapping file or a barcode fasta file, primers, etc get setup #dealing with Barcodes, get ion barcodes or parse the barcode_fasta argument barcode_file = os.path.join(base, base + ".barcodes_used.fa") rev_barcode_file = os.path.join(base, base + ".revbarcodes_used.fa") if os.path.isfile(barcode_file): os.remove(barcode_file) #check if mapping file passed, use this if present, otherwise use command line arguments SampleData = {} Barcodes = {} RevBarcodes = {} FwdPrimer = '' RevPrimer = '' if args.mapping_file: if not os.path.isfile(args.mapping_file): amptklib.log.error("Mapping file not found: %s" % args.mapping_file) sys.exit(1) SampleData, Barcodes, RevBarcodes, FwdPrimer, RevPrimer = amptklib.parseMappingFileNEW(args.mapping_file) else: if args.barcode_fasta: with open(barcode_file, 'w') as barcodeout: with open(args.barcode_fasta, 'r') as input: for rec in SeqIO.parse(input, 'fasta'): outname = args.multi+'.'+rec.id barcodeout.write(">%s\n%s\n" % (outname, rec.seq)) if args.reverse_barcode: with open(rev_barcode_file, 'w') as barcodeout: with open(args.reverse_barcode, 'r') as input: for rec in SeqIO.parse(input, 'fasta'): outname = args.multi+'.'+rec.id barcodeout.write(">%s\n%s\n" % (outname, rec.seq)) #parse primers here so doesn't conflict with mapping primers #look up primer db otherwise default to entry if FwdPrimer == '': if args.F_primer in amptklib.primer_db: FwdPrimer = amptklib.primer_db.get(args.F_primer) amptklib.log.info("{:} fwd primer found in AMPtk primer db, setting to: {:}".format(args.F_primer, FwdPrimer)) else: FwdPrimer = args.F_primer amptklib.log.info("{:} fwd primer not found in AMPtk primer db, assuming it is actual primer sequence.".format(args.F_primer)) if RevPrimer == '': if args.R_primer in amptklib.primer_db: RevPrimer = amptklib.primer_db.get(args.R_primer) amptklib.log.info("{:} rev primer found in AMPtk primer db, setting to: {:}".format(args.R_primer, RevPrimer)) else: RevPrimer = args.R_primer amptklib.log.info("{:} rev primer not found in AMPtk primer db, assuming it is actual primer sequence.".format(args.R_primer)) #then setup barcode dictionary if len(Barcodes) < 1 and os.path.isfile(barcode_file): Barcodes = amptklib.fasta2barcodes(barcode_file, False) #setup for looking for reverse barcode if len(RevBarcodes) < 1 and args.reverse_barcode: if not os.path.isfile(args.reverse_barcode): amptklib.log.info("Reverse barcode is not a valid file, exiting") sys.exit(1) shutil.copyfile(args.reverse_barcode, rev_barcode_file) RevBarcodes = amptklib.fasta2barcodes(rev_barcode_file, True) if args.platform != 'illumina': if not args.mapping_file and not args.barcode_fasta: amptklib.log.error("For ion, 454, or illumina2 datasets you must specificy a multi-fasta file containing barcodes with -b, --barcode_fasta, or -m/--mapping_file") sys.exit(1) if args.platform == 'illumina': #just need to get the correct .fastq.gz files into a folder by themselves #if illumina is selected, verify that args.fastq is a folder if not os.path.isdir(args.FASTQ): amptklib.log.error("%s is not a folder, for '--platform illumina', -i must be a folder containing raw reads" % (args.FASTQ)) sys.exit(1) rawlist = [] filelist = [] for file in os.listdir(args.FASTQ): if file.endswith(".fastq.gz") or file.endswith('.fastq') or file.endswith('.fq'): rawlist.append(file) if len(rawlist) > 0: if not '_R2' in sorted(rawlist)[1]: amptklib.log.info("Found %i single files, copying to %s folder" % (len(rawlist), base)) filelist = rawlist for file in rawlist: shutil.copyfile(os.path.join(args.FASTQ,file),(os.path.join(base,file))) else: amptklib.log.info("Found %i paired-end files, copying to %s folder" % (len(rawlist) / 2, base)) for file in rawlist: shutil.copyfile(os.path.join(args.FASTQ,file),(os.path.join(base,file))) if '_R1' in file: filelist.append(file) else: #start here to process the reads, first reverse complement the reverse primer ReverseCompRev = amptklib.RevComp(RevPrimer) #if --names given, load into dictonary if args.names: amptklib.log.info("Parsing names for output files via %s" % args.names) namesDict = {} with open(args.names, 'r') as input: for line in input: line = line.replace('\n', '') cols = line.split(',') if not cols[0] in namesDict: namesDict[cols[0]] = cols[1] #check for compressed input file if args.FASTQ.endswith('.gz'): amptklib.log.info("Gzipped input files detected, uncompressing") FASTQ_IN = args.FASTQ.replace('.gz', '') amptklib.Funzip(args.FASTQ, FASTQ_IN, multiprocessing.cpu_count()) else: FASTQ_IN = args.FASTQ #count FASTQ records in input amptklib.log.info("Loading FASTQ Records") total = amptklib.countfastq(FASTQ_IN) size = amptklib.checkfastqsize(args.FASTQ) readablesize = amptklib.convertSize(size) amptklib.log.info('{0:,}'.format(total) + ' reads (' + readablesize + ')') #output message depending on primer requirement if args.require_primer == 'off': amptklib.log.info("Looking for %i barcodes" % (len(Barcodes))) elif args.require_primer == 'forward': amptklib.log.info("Looking for %i barcodes that must have FwdPrimer: %s" % (len(Barcodes), FwdPrimer)) elif args.require_primer == 'both': amptklib.log.info("Looking for %i barcodes that must have FwdPrimer: %s and RevPrimer: %s" % (len(Barcodes), FwdPrimer, RevPrimer)) #this will loop through FASTQ file once, splitting those where barcodes are found, and primers trimmed runningTotal = 0 with open(FASTQ_IN, 'r') as input: for title, seq, qual in FastqGeneralIterator(input): Barcode, BarcodeLabel = amptklib.AlignBarcode(seq, Barcodes, args.barcode_mismatch) if Barcode == "": continue #trim barcode from sequence BarcodeLength = len(Barcode) seq = seq[BarcodeLength:] qual = qual[BarcodeLength:] #look for forward primer if args.require_primer != 'off': #means we only want ones with forward primer and or reverse, but don't remove them #now search for forward primer foralign = edlib.align(FwdPrimer, seq, mode="HW", k=args.primer_mismatch, additionalEqualities=amptklib.degenNuc) if foralign["editDistance"] < 0: continue if args.require_primer == 'both': #now search for reverse primer revalign = edlib.align(ReverseCompRev, seq, mode="HW", task="locations", k=args.primer_mismatch, additionalEqualities=amptklib.degenNuc) if revalign["editDistance"] < 0: #reverse primer was not found continue #check size if len(seq) < args.min_len: #filter out sequences less than minimum length. continue runningTotal += 1 fileout = os.path.join(base, BarcodeLabel+'.fastq') with open(fileout, 'a') as output: output.write("@%s\n%s\n+\n%s\n" % (title, seq, qual)) if args.require_primer == 'off': amptklib.log.info('{0:,}'.format(runningTotal) + ' total reads with valid barcode') elif args.require_primer == 'forward': amptklib.log.info('{0:,}'.format(runningTotal) + ' total reads with valid barcode and fwd primer') elif args.require_primer == 'both': amptklib.log.info('{0:,}'.format(runningTotal) + ' total reads with valid barcode and both primers') amptklib.log.info("Now Gzipping files") for file in os.listdir(base): if file.endswith(".fastq"): file_path = os.path.join(base, file) amptklib.Fzip_inplace(file_path) #after all files demuxed into output folder, loop through and create SRA metadata file filelist = [] for file in os.listdir(base): if file.endswith(".fastq.gz"): filelist.append(file) amptklib.log.info("Finished: output in %s" % base) #clean up if gzipped if args.FASTQ.endswith('.gz'): amptklib.removefile(FASTQ_IN) #check for BioSample meta file if args.biosample: amptklib.log.info("NCBI BioSample file detected, creating SRA metadata file") #load in BioSample file to dictionary with open(args.biosample, 'r') as input: reader = csv.reader(input, delimiter=str('\t')) header = next(reader) acc = header.index('Accession') sample = header.index('Sample Name') bio = header.index('BioProject') try: host = header.index('Host') except ValueError: host = header.index('Organism') BioDict = {col[sample]:(col[acc],col[bio],col[host]) for col in reader} #set some defaults based on the platform header = 'bioproject_accession\tbiosample_accession\tlibrary_ID\ttitle\tlibrary_strategy\tlibrary_source\tlibrary_selection\tlibrary_layout\tplatform\tinstrument_model\tdesign_description\tfiletype\tfilename\tfilename2\tforward_barcode\treverse_barcode\tforward_primer\treverse_primer\n' if args.platform == 'ion': sequencer = 'ION_TORRENT' model = 'Ion Torrent PGM' lib_layout = 'single' elif args.platform == '454': sequencer = '_LS454' model = '454 GS FLX Titanium' lib_layout = 'single' elif args.platform == 'illumina': sequencer = 'ILLUMINA' model = 'Illumina MiSeq' lib_layout = 'paired' else: amptklib.log.error("You specified a platform that is not supported") sys.exit(1) lib_strategy = 'AMPLICON' lib_source = 'GENOMIC' lib_selection = 'RANDOM PCR' filetype = 'fastq' #now open file for writing, input header and then loop through samples sub_out = base + '.submission.txt' with open(sub_out, 'w') as output: output.write(header) for file in filelist: barcode_for = '' barcode_rev = '' if not args.description: description = '%s amplicon library was created using a barcoded fusion primer PCR protocol using Pfx50 polymerase (Thermo Fisher Scientific), size selected, and sequenced on the %s platform. Sequence data was minimally processed, sequences were exported directly from the sequencing platform and only the barcode (index sequence) was trimmed prior to SRA submission. SRA submission generated with AMPtk %s' % (args.title, model, amptkversion.split(' ')[-1]) else: description = args.description if args.platform == 'ion' or args.platform == '454': name = file.split(".fastq")[0] if not name in BioDict: #lets try to look a bit harder, i.e. split on _ and - and look again searchname = name.replace('-', '_') searchname = searchname.split('_')[0] if not searchname in BioDict: #if still not found, then skip continue else: searchname = name bioproject = BioDict.get(searchname)[1] if not bioproject.startswith('PRJNA'): bioproject = 'PRJNA'+bioproject sample_name = BioDict.get(searchname)[0] title = '%s amplicon sequencing of %s: sample %s' % (args.title, BioDict.get(name)[2], name) bc_name = file.split(".f")[0] if bc_name in Barcodes: barcode_for = Barcodes.get(bc_name) if bc_name in RevBarcodes: barcode_rev = RevBarcodes.get(bc_name) if args.append: finalname = name+'_'+args.append #also need to change the name for output files newfile = file.replace(name, finalname) os.rename(os.path.join(base, file), os.path.join(base, newfile)) else: finalname = name newfile = file line = [bioproject,sample_name,finalname,title,lib_strategy,lib_source,lib_selection,lib_layout,sequencer,model,description,filetype,newfile,'',barcode_for,barcode_rev,FwdPrimer,RevPrimer] elif args.platform == 'illumina': name = file.split("_")[0] if not name in BioDict: amptklib.log.info('{:} not found in BioSample text file'.format(name)) continue bioproject = BioDict.get(name)[1] if not bioproject.startswith('PRJNA'): bioproject = 'PRJNA'+bioproject sample_name = BioDict.get(name)[0] title = '%s amplicon sequencing of %s: sample %s' % (args.title, BioDict.get(name)[2], name) file2 = file.replace('_R1', '_R2') #count number of _ in name, determines the dataformat fields = file.count("_") if fields > 3: #this is full illumina name with dual barcodes dualBC = file.split("_")[1] if '-' in dualBC: barcode_for = dualBC.split('-')[0] barcode_rev = dualBC.split('-')[1] elif fields == 3: #this is older reverse barcoded name barcode_for = '' barcode_rev = file.split("_")[1] if args.append: finalname = name+'_'+args.append newfile = file.replace(name, finalname) newfile2 = file2.replace(name, finalname) #also need to change the name for output files os.rename(os.path.join(base, file), os.path.join(base, newfile1)) os.rename(os.path.join(base, file2), os.path.join(base, newfile2)) file = file.replace(name, finalname) else: finalname = name newfile = file newfile2 = file2 line = [bioproject,sample_name,finalname,title,lib_strategy,lib_source,lib_selection,lib_layout,sequencer,model,description,filetype,newfile,newfile2,barcode_for,barcode_rev,FwdPrimer,RevPrimer] #write output to file output.write('\t'.join(line)+'\n') amptklib.log.info("SRA submission file created: %s" % sub_out)
def main(args):
global FwdPrimer, RevPrimer, Barcodes, tmpdir
parser = argparse.ArgumentParser(
prog='amptk-process_ion.py',
usage="%(prog)s [options] -i file.fastq\n%(prog)s -h for help menu",
description=
'''Script finds barcodes, strips forward and reverse primers, relabels, and then trim/pads reads to a set length''',
epilog="""Written by Jon Palmer (2015) [email protected]""",
formatter_class=MyFormatter)
parser.add_argument('-i',
'--fastq',
'--sff',
'--fasta',
'--bam',
dest='fastq',
required=True,
help='BAM/FASTQ/SFF/FASTA file')
parser.add_argument('-q', '--qual', help='QUAL file (if -i is FASTA)')
parser.add_argument('-o',
'--out',
dest="out",
default='ion',
help='Base name for output')
parser.add_argument('-f',
'--fwd_primer',
dest="F_primer",
default='fITS7-ion',
help='Forward Primer')
parser.add_argument('-r',
'--rev_primer',
dest="R_primer",
default='ITS4',
help='Reverse Primer')
parser.add_argument(
'-m',
'--mapping_file',
help='Mapping file: QIIME format can have extra meta data columns')
parser.add_argument('-p',
'--pad',
default='off',
choices=['on', 'off'],
help='Pad with Ns to a set length')
parser.add_argument('--primer_mismatch',
default=2,
type=int,
help='Number of mis-matches in primer')
parser.add_argument('--barcode_mismatch',
default=0,
type=int,
help='Number of mis-matches in barcode')
parser.add_argument(
'--barcode_fasta',
default='ionxpress',
help='FASTA file containing Barcodes (Names & Sequences)')
parser.add_argument('--reverse_barcode',
help='FASTA file containing 3 prime Barocdes')
parser.add_argument('-b',
'--list_barcodes',
dest="barcodes",
default='all',
help='Enter Barcodes used separated by commas')
parser.add_argument('--min_len',
default=100,
type=int,
help='Minimum read length to keep')
parser.add_argument('-l',
'--trim_len',
default=300,
type=int,
help='Trim length for reads')
parser.add_argument(
'--full_length',
action='store_true',
help='Keep only full length reads (no trimming/padding)')
parser.add_argument('--mult_samples',
dest="multi",
default='False',
help='Combine multiple samples (i.e. FACE1)')
parser.add_argument('--ion',
action='store_true',
help='Input data is Ion Torrent')
parser.add_argument('--454', action='store_true', help='Input data is 454')
parser.add_argument('--cpus',
type=int,
help="Number of CPUs. Default: auto")
parser.add_argument('-u',
'--usearch',
dest="usearch",
default='usearch9',
help='USEARCH EXE')
args = parser.parse_args(args)
args.out = re.sub(r'\W+', '', args.out)
log_name = args.out + '.amptk-demux.log'
if os.path.isfile(log_name):
os.remove(log_name)
FNULL = open(os.devnull, 'w')
amptklib.setupLogging(log_name)
cmd_args = " ".join(sys.argv) + '\n'
amptklib.log.debug(cmd_args)
print("-------------------------------------------------------")
#initialize script, log system info and usearch version
amptklib.SystemInfo()
#Do a version check
usearch = args.usearch
amptklib.versionDependencyChecks(usearch)
#get number of CPUs to use
if not args.cpus:
cpus = multiprocessing.cpu_count()
else:
cpus = args.cpus
#parse a mapping file or a barcode fasta file, primers, etc get setup
#dealing with Barcodes, get ion barcodes or parse the barcode_fasta argument
barcode_file = args.out + ".barcodes_used.fa"
rev_barcode_file = args.out + '.revbarcodes_used.fa'
amptklib.SafeRemove(barcode_file)
amptklib.SafeRemove(rev_barcode_file)
#check if mapping file passed, use this if present, otherwise use command line arguments
SampleData = {}
Barcodes = {}
RevBarcodes = {}
if args.mapping_file:
if not os.path.isfile(args.mapping_file):
amptklib.log.error("Mapping file not found: %s" %
args.mapping_file)
sys.exit(1)
SampleData, Barcodes, RevBarcodes, FwdPrimer, RevPrimer = amptklib.parseMappingFileNEW(
args.mapping_file)
genericmapfile = args.mapping_file
else: #no mapping file, so create dictionaries from barcode fasta files
if args.barcode_fasta == 'ionxpress':
#get script path and barcode file name
pgm_barcodes = os.path.join(os.path.dirname(amptklib.__file__),
'DB', 'ionxpress_barcodes.fa')
elif args.barcode_fasta == 'ioncode':
pgm_barcodes = os.path.join(os.path.dirname(amptklib.__file__),
'DB', 'ioncode_barcodes.fa')
if args.barcode_fasta == 'ionxpress' or args.barcode_fasta == 'ioncode':
if args.barcodes == "all":
if args.multi == 'False':
shutil.copyfile(pgm_barcodes, barcode_file)
else:
with open(barcode_file, 'w') as barcodeout:
with open(pgm_barcodes, 'r') as input:
for rec in SeqIO.parse(input, 'fasta'):
outname = args.multi + '.' + rec.id
barcodeout.write(">%s\n%s\n" %
(outname, rec.seq))
else:
bc_list = args.barcodes.split(",")
inputSeqFile = open(pgm_barcodes, "rU")
SeqRecords = SeqIO.to_dict(SeqIO.parse(inputSeqFile, "fasta"))
for rec in bc_list:
name = "BC." + rec
seq = SeqRecords[name].seq
if args.multi != 'False':
outname = args.multi + '.' + name
else:
outname = name
outputSeqFile = open(barcode_file, "a")
outputSeqFile.write(">%s\n%s\n" % (outname, seq))
outputSeqFile.close()
inputSeqFile.close()
else:
#check for multi_samples and add if necessary
if args.multi == 'False':
shutil.copyfile(args.barcode_fasta, barcode_file)
if args.reverse_barcode:
shutil.copyfile(args.reverse_barcode, rev_barcode_file)
else:
with open(barcode_file, 'w') as barcodeout:
with open(args.barcode_fasta, 'r') as input:
for rec in SeqIO.parse(input, 'fasta'):
outname = args.multi + '.' + rec.id
barcodeout.write(">%s\n%s\n" % (outname, rec.seq))
if args.reverse_barcode:
with open(rev_barcode_file, 'w') as barcodeout:
with open(args.reverse_barcode, 'r') as input:
for rec in SeqIO.parse(input, 'fasta'):
outname = args.multi + '.' + rec.id
barcodeout.write(">%s\n%s\n" %
(outname, rec.seq))
#parse primers here so doesn't conflict with mapping primers
#look up primer db otherwise default to entry
if args.F_primer in amptklib.primer_db:
FwdPrimer = amptklib.primer_db.get(args.F_primer)
amptklib.log.info(
"{:} fwd primer found in AMPtk primer db, setting to: {:}".
format(args.F_primer, FwdPrimer))
else:
FwdPrimer = args.F_primer
amptklib.log.info(
"{:} fwd primer not found in AMPtk primer db, assuming it is actual primer sequence."
.format(args.F_primer))
if args.R_primer in amptklib.primer_db:
RevPrimer = amptklib.primer_db.get(args.R_primer)
amptklib.log.info(
"{:} rev primer found in AMPtk primer db, setting to: {:}".
format(args.R_primer, RevPrimer))
else:
RevPrimer = args.R_primer
amptklib.log.info(
"{:} rev primer not found in AMPtk primer db, assuming it is actual primer sequence."
.format(args.R_primer))
#check if input is compressed
gzip_list = []
if args.fastq.endswith('.gz'):
gzip_list.append(os.path.abspath(args.fastq))
if gzip_list:
amptklib.log.info("Gzipped input files detected, uncompressing")
for file in gzip_list:
file_out = file.replace('.gz', '')
amptklib.Funzip(file, file_out, cpus)
args.fastq = args.fastq.replace('.gz', '')
#if SFF file passed, convert to FASTQ with biopython
if args.fastq.endswith(".sff"):
if args.barcode_fasta == 'ionxpress':
if not args.mapping_file:
amptklib.log.error(
"You did not specify a --barcode_fasta or --mapping_file, one is required for 454 data"
)
sys.exit(1)
amptklib.log.info("SFF input detected, converting to FASTQ")
SeqIn = args.out + '.sff.extract.fastq'
SeqIO.convert(args.fastq, "sff-trim", SeqIn, "fastq")
elif args.fastq.endswith(".fas") or args.fastq.endswith(
".fasta") or args.fastq.endswith(".fa"):
if not args.qual:
amptklib.log.error(
"FASTA input detected, however no QUAL file was given. You must have FASTA + QUAL files"
)
sys.exit(1)
else:
if args.barcode_fasta == 'ionxpress':
if not args.mapping_file:
amptklib.log.error(
"You did not specify a --barcode_fasta or --mapping_file, one is required for 454 data"
)
sys.exit(1)
SeqIn = args.out + '.fastq'
amptklib.log.info("FASTA + QUAL detected, converting to FASTQ")
amptklib.faqual2fastq(args.fastq, args.qual, SeqIn)
elif args.fastq.endswith('.bam'):
#so we can convert natively with pybam, however it is 10X slower than bedtools/samtools
#since samtools is fastest, lets use that if exists, if not then bedtools, else default to pybam
amptklib.log.info("Converting Ion Torrent BAM file to FASTQ")
SeqIn = args.out + '.fastq'
if amptklib.which('samtools'):
cmd = ['samtools', 'fastq', '-@', str(cpus), args.fastq]
amptklib.runSubprocess2(cmd, amptklib.log, SeqIn)
else:
if amptklib.which('bedtools'):
cmd = [
'bedtools', 'bamtofastq', '-i', args.fastq, '-fq', SeqIn
]
amptklib.runSubprocess(cmd, amptklib.log)
else: #default to pybam
amptklib.bam2fastq(args.fastq, SeqIn)
else:
SeqIn = args.fastq
#start here to process the reads, first reverse complement the reverse primer
catDemux = args.out + '.demux.fq'
origRevPrimer = RevPrimer
RevPrimer = amptklib.RevComp(RevPrimer)
amptklib.log.info("Foward primer: %s, Rev comp'd rev primer: %s" %
(FwdPrimer, RevPrimer))
#then setup barcode dictionary
if len(Barcodes) < 1:
Barcodes = amptklib.fasta2barcodes(barcode_file, False)
#setup for looking for reverse barcode
if len(RevBarcodes) < 1 and args.reverse_barcode:
if not os.path.isfile(args.reverse_barcode):
amptklib.log.info("Reverse barcode is not a valid file, exiting")
sys.exit(1)
shutil.copyfile(args.reverse_barcode, rev_barcode_file)
RevBarcodes = amptklib.fasta2barcodes(rev_barcode_file, True)
#Count FASTQ records
amptklib.log.info("Loading FASTQ Records")
orig_total = amptklib.countfastq(SeqIn)
size = amptklib.checkfastqsize(SeqIn)
readablesize = amptklib.convertSize(size)
amptklib.log.info('{0:,}'.format(orig_total) + ' reads (' + readablesize +
')')
#create tmpdir and split input into n cpus
tmpdir = args.out.split('.')[0] + '_' + str(os.getpid())
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
amptklib.log.info(
'Dropping reads less than {:} bp and setting lossless trimming to {:} bp.'
.format(args.min_len, args.trim_len))
if cpus > 1:
#split fastq file
amptklib.log.info("Splitting FASTQ files over {:} cpus".format(cpus))
amptklib.split_fastq(SeqIn, orig_total, tmpdir, cpus * 2)
#now get file list from tmp folder
file_list = []
for file in os.listdir(tmpdir):
if file.endswith(".fq"):
file = os.path.join(tmpdir, file)
file_list.append(file)
#finally process reads over number of cpus
amptklib.runMultiProgress(processRead, file_list, cpus, args=args)
else:
shutil.copyfile(SeqIn, os.path.join(tmpdir, 'chunk.fq'))
processRead(os.path.join(tmpdir, 'chunk.fq'), args=args)
print("-------------------------------------------------------")
#Now concatenate all of the demuxed files together
amptklib.log.info("Concatenating Demuxed Files")
tmpDemux = args.out + '.tmp.demux.fq'
with open(tmpDemux, 'w') as outfile:
for filename in glob.glob(os.path.join(tmpdir, '*.demux.fq')):
if filename == tmpDemux:
continue
with open(filename, 'r') as readfile:
shutil.copyfileobj(readfile, outfile)
#parse the stats
finalstats = [0, 0, 0, 0, 0, 0, 0]
for file in os.listdir(tmpdir):
if file.endswith('.stats'):
with open(os.path.join(tmpdir, file), 'r') as statsfile:
line = statsfile.readline()
line = line.rstrip()
newstats = line.split(',')
newstats = [int(i) for i in newstats]
for x, num in enumerate(newstats):
finalstats[x] += num
#clean up tmp folder
shutil.rmtree(tmpdir)
#last thing is to re-number of reads as it is possible they could have same name from multitprocessor split
amptklib.fastqreindex(tmpDemux, catDemux)
os.remove(tmpDemux)
amptklib.log.info('{0:,}'.format(finalstats[0]) + ' total reads')
if args.reverse_barcode:
amptklib.log.info('{0:,}'.format(finalstats[0] - finalstats[1] -
finalstats[2] - finalstats[4]) +
' valid Fwd and Rev Barcodes')
else:
amptklib.log.info('{0:,}'.format(finalstats[0] - finalstats[1]) +
' valid Barcode')
amptklib.log.info('{0:,}'.format(finalstats[0] - finalstats[1] -
finalstats[2]) +
' Fwd Primer found, {0:,}'.format(finalstats[3]) +
' Rev Primer found')
amptklib.log.info('{0:,}'.format(finalstats[5]) +
' discarded too short (< %i bp)' % args.min_len)
amptklib.log.info('{0:,}'.format(finalstats[6]) + ' valid output reads')
#now loop through data and find barcoded samples, counting each.....
BarcodeCount = {}
with open(catDemux, 'r') as input:
header = itertools.islice(input, 0, None, 4)
for line in header:
ID = line.split("=", 1)[-1].split(";")[0]
if ID not in BarcodeCount:
BarcodeCount[ID] = 1
else:
BarcodeCount[ID] += 1
#now let's count the barcodes found and count the number of times they are found.
barcode_counts = "%22s: %s" % ('Sample', 'Count')
for k, v in natsorted(list(BarcodeCount.items()),
key=lambda k_v: k_v[1],
reverse=True):
barcode_counts += "\n%22s: %s" % (k, str(BarcodeCount[k]))
amptklib.log.info("Found %i barcoded samples\n%s" %
(len(BarcodeCount), barcode_counts))
#create a generic mappingfile for downstream processes
genericmapfile = args.out + '.mapping_file.txt'
if not args.mapping_file:
amptklib.CreateGenericMappingFile(Barcodes, RevBarcodes, FwdPrimer,
origRevPrimer, genericmapfile,
BarcodeCount)
else:
amptklib.updateMappingFile(args.mapping_file, BarcodeCount,
genericmapfile)
#compress the output to save space
FinalDemux = catDemux + '.gz'
amptklib.Fzip(catDemux, FinalDemux, cpus)
amptklib.removefile(catDemux)
if gzip_list:
for file in gzip_list:
file = file.replace('.gz', '')
amptklib.removefile(file)
#get file size
filesize = os.path.getsize(FinalDemux)
readablesize = amptklib.convertSize(filesize)
amptklib.log.info("Output file: %s (%s)" % (FinalDemux, readablesize))
amptklib.log.info("Mapping file: %s" % genericmapfile)
print("-------------------------------------------------------")
if 'darwin' in sys.platform:
print(col.WARN + "\nExample of next cmd: " + col.END +
"amptk cluster -i %s -o out\n" % (FinalDemux))
else:
print("\nExample of next cmd: amptk cluster -i %s -o out\n" %
(FinalDemux))