def align_reads(
fq1=None,
fq2=None,
fqU=None,
ref_fa=None,
outdir='.',
bt2_preset='sensitive-local',
sample_id='sampleXX',
no_realign=False,
remove_duplicates=False,
encoding=None,
ncpu=1,
xmx=sysutils.get_java_heap_size(),
keep_tmp=False,
quiet=False,
logfile=None,
debug=False,
):
""" Pipeline step to align reads
Args:
fq1 (str): Path to fastq file with read 1
fq2 (str): Path to fastq file with read 2
fqU (str): Path to fastq file with unpaired reads
ref_fa (str): Path to reference fasta file
outdir (str): Path to output directory
bt2_preset (str): Bowtie2 preset to use for alignment
sample_id (str): Read group ID
no_realign (bool): Do not realign indels
remove_duplicates (bool): Remove duplicates from final alignment
encoding (str): Quality score encoding
ncpu (int): Number of CPUs to use
xmx (int): Maximum heap size for JVM in GB
keep_tmp (bool): Do not delete temporary directory
quiet (bool): Do not write output to console
logfile (file): Append console output to this file
debug (bool): Print commands but do not run
Returns:
out_aligned (str): Path to aligned BAM file
out_bt2 (str): Path to bowtie2 report
"""
# Check inputs
if fq1 is not None and fq2 is not None and fqU is None:
input_reads = "paired" # Paired end
elif fq1 is None and fq2 is None and fqU is not None:
input_reads = "single" # Single end
elif fq1 is not None and fq2 is not None and fqU is not None:
input_reads = "both"
else:
msg = "incorrect input reads; requires either "
msg += "(--fq1 AND --fq2) OR (--fqU) OR (--fq1 AND --fq2 AND --fqU)"
raise MissingRequiredArgument(msg)
if encoding is None:
if input_reads == 'single':
encoding = helpers.guess_encoding(fqU)
else:
encoding = helpers.guess_encoding(fq1)
# Check dependencies
sysutils.check_dependency('bowtie2')
sysutils.check_dependency('samtools')
sysutils.check_dependency('picard')
# Identify correct command for GATK
GATK_BIN = sysutils.determine_dependency_path(['gatk', 'gatk3'])
# Set JVM heap argument (for GATK)
JAVA_HEAP = '_JAVA_OPTIONS="-Xmx%dg"' % xmx
# Outputs
out_aligned = os.path.join(outdir, 'aligned.bam')
out_bt2 = os.path.join(outdir, 'aligned.bt2.out')
# Temporary directory
tempdir = sysutils.create_tempdir('align_reads', None, quiet, logfile)
# Copy and index initial reference
curref = os.path.join(tempdir, 'initial.fasta')
cmd1 = ['cp', ref_fa, curref]
cmd2 = ['samtools', 'faidx', curref]
cmd3 = [
'picard', 'CreateSequenceDictionary',
'R=%s' % curref,
'O=%s' % os.path.join(tempdir, 'initial.dict')
]
cmd4 = ['bowtie2-build', curref, os.path.join(tempdir, 'initial')]
sysutils.command_runner([cmd1, cmd2, cmd3, cmd4], 'align_reads:index',
quiet, logfile, debug)
# Align with bowtie2
cmd5 = [
'bowtie2',
'-p',
'%d' % ncpu,
'--phred33' if encoding == "Phred+33" else '--phred64',
'--no-unal',
'--rg-id',
sample_id,
'--rg',
'SM:%s' % sample_id,
'--rg',
'LB:1',
'--rg',
'PU:1',
'--rg',
'PL:illumina',
'--%s' % bt2_preset,
'-x',
'%s' % os.path.join(tempdir, 'initial'),
]
if input_reads in [
'paired',
'both',
]:
cmd5 += [
'-1',
fq1,
'-2',
fq2,
]
elif input_reads in [
'single',
'both',
]:
cmd5 += [
'-U',
fqU,
]
cmd5 += [
'-S',
os.path.join(tempdir, 'aligned.bt2.sam'),
]
cmd5 += [
'2>',
out_bt2,
]
try:
sysutils.command_runner([
cmd5,
], 'align_reads:bowtie2', quiet, logfile, debug)
except PipelineStepError as e:
if os.path.exists(out_bt2):
with open(out_bt2, 'r') as fh:
print('[--- bowtie2 stderr ---]\n%s' % fh.read(),
file=sys.stderr)
raise
cmd6 = [
'samtools',
'view',
'-u',
os.path.join(tempdir, 'aligned.bt2.sam'),
'|',
'samtools',
'sort',
'>',
os.path.join(tempdir, 'sorted.bam'),
]
cmd7 = [
'samtools',
'index',
os.path.join(tempdir, 'sorted.bam'),
]
sysutils.command_runner([
cmd6,
cmd7,
], 'align_reads:samsort', quiet, logfile, debug)
cur_bam = os.path.join(tempdir, 'sorted.bam')
if remove_duplicates:
sysutils.log_message('[--- Removing duplicates ---]', quiet, logfile)
else:
sysutils.log_message('[--- Marking duplicates ---]', quiet, logfile)
# MarkDuplicates
cmd8 = [
'picard',
'MarkDuplicates',
'CREATE_INDEX=true',
'USE_JDK_DEFLATER=true',
'USE_JDK_INFLATER=true',
'M=%s' % os.path.join(tempdir, 'rmdup.metrics.txt'),
'I=%s' % cur_bam,
'O=%s' % os.path.join(tempdir, 'rmdup.bam'),
]
if remove_duplicates:
cmd8 += [
'REMOVE_DUPLICATES=true',
]
sysutils.command_runner([
cmd8,
], 'align_reads:markdups', quiet, logfile, debug)
cur_bam = os.path.join(tempdir, 'rmdup.bam')
if no_realign:
print('[--- Skipping realignment ---]', file=sys.stderr)
else:
# RealignerTargetCreator
cmd9 = [
JAVA_HEAP,
GATK_BIN,
'-T',
'RealignerTargetCreator',
'-I',
cur_bam,
'-R',
curref,
'-o',
os.path.join(tempdir, 'tmp.intervals'),
]
# IndelRealigner
cmd10 = [
JAVA_HEAP, GATK_BIN, '-T', 'IndelRealigner', '--use_jdk_deflater',
'--use_jdk_inflater', '-maxReads', '1000000', '-dt', 'NONE', '-I',
cur_bam, '-R', curref, '-targetIntervals',
os.path.join(tempdir, 'tmp.intervals'), '-o',
os.path.join(tempdir, 'realign.bam')
]
sysutils.command_runner([
cmd9,
cmd10,
], 'align_reads:realign', quiet, logfile, debug)
cur_bam = os.path.join(tempdir, 'realign.bam')
# Check that cur_bam was created
if not os.path.exists(cur_bam):
msg = "BAM does not exist: %s" % cur_bam
raise sysutils.PipelineStepError(msg)
cmd11a = [
'rm',
'-f',
out_aligned,
]
cmd11b = [
'mv',
cur_bam,
out_aligned,
]
cmd11c = [
'samtools',
'index',
out_aligned,
]
sysutils.command_runner([
cmd11a,
cmd11b,
cmd11c,
], 'align_reads:copy', quiet, logfile, debug)
if not keep_tmp:
sysutils.remove_tempdir(tempdir, 'align_reads', quiet, logfile)
return out_aligned, out_bt2
def ec_reads(
fq1=None,
fq2=None,
fqU=None,
outdir='.',
ncpu=1,
keep_tmp=False,
quiet=False,
logfile=None,
debug=False,
):
""" Pipeline step to error-correct reads using spades
Args:
fq1 (str): Path to fastq file with read 1
fq2 (str): Path to fastq file with read 2
fqU (str): Path to fastq file with unpaired reads
outdir (str): Path to output directory
ncpu (int): Number of CPUs to use
keep_tmp (bool): Do not delete temporary directory
quiet (bool): Do not write output to console
logfile (file): Append console output to this file
debug (bool): Print commands but do not run
Returns:
out1 (str): Path to corrected fastq file with read 1
out2 (str): Path to corrected fastq file with read 2
outU (str): Path to corrected fastq file with unpaired reads
"""
# Check inputs
if fq1 is not None and fq2 is not None and fqU is None:
input_reads = "paired" # Paired end
elif fq1 is None and fq2 is None and fqU is not None:
input_reads = "single" # Single end
elif fq1 is not None and fq2 is not None and fqU is not None:
input_reads = "both"
else:
msg = "incorrect input reads; requires either "
msg += "(--fq1 AND --fq2) OR (--fqU) OR (--fq1 AND --fq2 AND --fqU)"
raise MissingRequiredArgument(msg)
# Check dependencies
sysutils.check_dependency('spades.py')
# Outputs
out1 = os.path.join(outdir, 'corrected_1.fastq')
out2 = os.path.join(outdir, 'corrected_2.fastq')
outU = os.path.join(outdir, 'corrected_U.fastq')
# Temporary directory
tempdir = sysutils.create_tempdir('ec_reads', None, quiet, logfile)
# spades command
cmd1 = [
'spades.py',
'-o',
tempdir,
'-t',
'%d' % ncpu,
'--only-error-correction',
]
if input_reads in [
'paired',
'both',
]:
cmd1 += [
'-1',
os.path.abspath(fq1),
'-2',
os.path.abspath(fq2),
]
if input_reads in [
'single',
'both',
]:
cmd1 += [
'-s',
os.path.abspath(fqU),
]
sysutils.command_runner([
cmd1,
], 'ec_reads', quiet, logfile, debug)
# Copy files
yaml_file = os.path.join(tempdir, 'corrected/corrected.yaml')
if not os.path.exists(yaml_file):
sysutils.PipelineStepError("YAML file %s not found" % yaml_file)
with open(yaml_file, 'rU') as fh:
d = yaml.load(fh, Loader=yaml.FullLoader)[0]
cmds = []
if 'left reads' in d:
cmds.append([
'gunzip',
'-c',
] + sorted(d['left reads']) + ['>', out1])
if 'right reads' in d:
cmds.append([
'gunzip',
'-c',
] + sorted(d['right reads']) + ['>', out2])
if 'single reads' in d:
cmds.append([
'gunzip',
'-c',
] + sorted(d['single reads']) + ['>', outU])
sysutils.command_runner(cmds, 'ec_reads', quiet, logfile, debug)
if not keep_tmp:
sysutils.remove_tempdir(tempdir, 'ec_reads', quiet, logfile)
return out1, out2, outU
def trim_reads(
fq1=None,
fq2=None,
fqU=None,
outdir=".",
adapter_file=None,
trimmers=TRIMMERS,
encoding=None,
ncpu=1,
quiet=False,
logfile=None,
debug=False,
):
""" Pipeline step to trim reads
Args:
fq1 (str): Path to fastq file with read 1
fq2 (str): Path to fastq file with read 2
fqU (str): Path to fastq file with unpaired reads
outdir (str): Path to output directory
adapter_file (str): Path to adapter file (fasta)
trimmers (`list` of `str`): Trim commands for trimmomatic
encoding (str): Quality score encoding
ncpu (int): Number of CPUs to use
quiet (bool): Do not write output to console
logfile (file): Append console output to this file
debug (bool): Print commands but do not run
Returns:
out1 (str): Path to trimmed fastq file with read 1
out2 (str): Path to trimmed fastq file with read 2
outU (str): Path to trimmed fastq file with unpaired reads
out_summary (str): Path to summary file
"""
# Check inputs
if fq1 is not None and fq2 is not None and fqU is None:
input_reads = "paired" # Paired end
elif fq1 is None and fq2 is None and fqU is not None:
input_reads = "single" # Single end
else:
msg = "incorrect input reads; requires either "
msg += "(--fq1 and --fq2) OR (--fqU)"
raise MissingRequiredArgument(msg)
""" There are two different ways to call Trimmomatic. If using modules on
C1, the path to the jar file is stored in the "$Trimmomatic"
environment variable. Otherwise, if using conda, the "trimmomatic"
script is in PATH.
"""
# Check dependencies
try:
sysutils.check_dependency('trimmomatic')
cmd1 = ['trimmomatic']
except PipelineStepError as e:
if 'Trimmomatic' in os.environ:
cmd1 = ['java', '-jar', '$Trimmomatic']
else:
raise e
# Get encoding
if encoding is None:
if input_reads == 'single':
encoding = helpers.guess_encoding(fqU)
else:
encoding = helpers.guess_encoding(fq1)
# Outputs for both single and paired
out_summary = os.path.join(outdir, 'trimmomatic_summary.out')
outU = os.path.join(outdir, 'trimmed_U.fastq')
if input_reads is 'single':
# Outputs
out1 = out2 = None
# Trimmomatic command
cmd1 += [
'SE',
'-threads',
'%d' % ncpu,
'-phred33' if encoding == "Phred+33" else '-phred64',
'-summary',
out_summary,
fqU,
outU,
]
# Specify trimming steps
if adapter_file is not None:
adapter_file = adapter_file.replace('PE', 'SE')
cmd1.append("ILLUMINACLIP:%s:2:30:10" % adapter_file)
cmd1 += trimmers
# Run command
sysutils.command_runner([
cmd1,
], 'trim_reads', quiet, logfile, debug)
return out1, out2, outU
elif input_reads is 'paired':
# Outputs
out1 = os.path.join(outdir, 'trimmed_1.fastq')
out2 = os.path.join(outdir, 'trimmed_2.fastq')
tmp1U = os.path.join(outdir, 'tmp1U.fq')
tmp2U = os.path.join(outdir, 'tmp2U.fq')
# Trimmomatic command
cmd1 += [
'PE',
'-threads',
'%d' % ncpu,
'-phred33' if encoding == "Phred+33" else '-phred64',
'-summary',
out_summary,
fq1,
fq2,
out1,
tmp1U,
out2,
tmp2U,
]
# Specify trimming steps
if adapter_file is not None:
cmd1.append("ILLUMINACLIP:%s:2:30:10" % adapter_file)
cmd1 += trimmers
# Concat files command
cmd2 = [
'cat',
tmp1U,
tmp2U,
'>>',
outU,
]
cmd3 = [
'rm',
'-f',
tmp1U,
tmp2U,
]
# Run commands
sysutils.command_runner([
cmd1,
cmd2,
cmd3,
], 'trim_reads', quiet, logfile, debug)
return out1, out2, outU, out_summary
def assemble_denovo_trinity(fq1=None,
fq2=None,
fqU=None,
outdir='.',
min_contig_length=200,
subsample=None,
seed=None,
ncpu=1,
keep_tmp=False,
quiet=False,
logfile=None,
debug=False,
**kwargs):
""" Pipeline step to assemble reads using Trinity (denovo)
Args:
fq1 (str): Path to fastq file with read 1
fq2 (str): Path to fastq file with read 2
fqU (str): Path to fastq file with unpaired reads
outdir (str): Path to output directory
min_contig_length (int): minimum assembled contig length to report
subsample (int): use a subsample of reads for assembly
seed (int): Seed for random number generator
ncpu (int): Number of CPUs to use
keep_tmp (bool): Do not delete temporary directory
quiet (bool): Do not write output to console
logfile (file): Append console output to this file
debug (bool): Print commands but do not run
**kwargs: Not used.
Returns:
out1 (str): Path to assembled contigs file (fasta format)
"""
# Check inputs
if fq1 is not None and fq2 is not None and fqU is None:
input_reads = "paired" # Paired end
elif fq1 is None and fq2 is None and fqU is not None:
input_reads = "single" # Single end
elif fq1 is not None and fq2 is not None and fqU is not None:
input_reads = "both"
else:
msg = "incorrect input reads; requires either "
msg += "(--fq1 AND --fq2) OR (--fqU) OR (--fq1 AND --fq2 AND --fqU)"
raise MissingRequiredArgument(msg)
# Check dependencies
sysutils.check_dependency('Trinity')
# Outputs
out1 = os.path.join(outdir, 'contigs.fa')
# Temporary directory
tempdir = sysutils.create_tempdir('assemble_trinity', None, quiet, logfile)
# Trinity command
cmd1 = [
'Trinity',
'--min_contig_length',
'%d' % min_contig_length,
'--CPU',
'%d' % ncpu,
#'--max_memory', '%dG' % max_memory,
'--seqType',
'fq',
'--output',
tempdir,
]
if input_reads in [
'paired',
'both',
]:
cmd1 += [
'--left',
os.path.abspath(fq1),
'--right',
os.path.abspath(fq2),
]
elif input_reads in [
'single',
'both',
]:
cmd1 += [
'--single',
os.path.abspath(fqU),
]
# Copy command
cmd2 = [
'cp',
os.path.join(tempdir, 'Trinity.fasta'),
out1,
]
sysutils.command_runner([
cmd1,
cmd2,
], 'assemble_trinity', quiet, logfile, debug)
if not keep_tmp:
sysutils.remove_tempdir(tempdir, 'assemble_trinity', quiet, logfile)
if os.path.isfile(out1):
with open(os.path.join(outdir, 'assembly_summary.txt'), 'w') as outh:
sequtils.assembly_stats(open(out1, 'rU'), outh)
return out1
def assemble_denovo_spades(fq1=None,
fq2=None,
fqU=None,
outdir='.',
no_error_correction=False,
subsample=None,
seed=None,
ncpu=1,
keep_tmp=False,
quiet=False,
logfile=None,
debug=False,
**kwargs):
""" Pipeline step to assemble reads using spades (denovo)
Args:
fq1 (str): Path to fastq file with read 1
fq2 (str): Path to fastq file with read 2
fqU (str): Path to fastq file with unpaired reads
outdir (str): Path to output directory
no_error_correction (bool): do not perform error correction
subsample (int): use a subsample of reads for assembly
seed (int): Seed for random number generator
ncpu (int): Number of CPUs to use
keep_tmp (bool): Do not delete temporary directory
quiet (bool): Do not write output to console
logfile (file): Append console output to this file
debug (bool): Print commands but do not run
**kwargs: Not used.
Returns:
out_fa (str): Path to assembled contigs file (fasta format)
out_summary (str): Path to assembly summary
"""
# Check inputs
if fq1 is not None and fq2 is not None and fqU is None:
input_reads = "paired" # Paired end
elif fq1 is None and fq2 is None and fqU is not None:
input_reads = "single" # Single end
elif fq1 is not None and fq2 is not None and fqU is not None:
input_reads = "both"
else:
msg = "incorrect input reads; requires either "
msg += "(--fq1 AND --fq2) OR (--fqU) OR (--fq1 AND --fq2 AND --fqU)"
raise MissingRequiredArgument(msg)
# Check dependencies
sysutils.check_dependency('spades.py')
# Outputs
out_fa = os.path.join(outdir, 'denovo_contigs.fna')
out_summary = os.path.join(outdir, 'denovo_summary.txt')
# Temporary directory
tempdir = sysutils.create_tempdir('assemble_spades', None, quiet, logfile)
# Subsample
if subsample is not None:
full1, full2, fullU = fq1, fq2, fqU
fq1, fq2, fqU = sample_reads.sample_reads(fq1=full1,
fq2=full2,
fqU=fullU,
outdir=tempdir,
nreads=subsample,
seed=seed,
quiet=quiet,
logfile=logfile,
debug=debug)
# spades command
cmd1 = [
'spades.py',
'-o',
tempdir,
'-t',
'%d' % ncpu,
]
if input_reads in [
'paired',
'both',
]:
cmd1 += [
'-1',
os.path.abspath(fq1),
'-2',
os.path.abspath(fq2),
]
if input_reads in [
'single',
'both',
]:
cmd1 += [
'-s',
os.path.abspath(fqU),
]
if no_error_correction:
cmd1 += [
'--only-assembler',
]
sysutils.command_runner([
cmd1,
], 'assemble_spades', quiet, logfile, debug)
shutil.copy(os.path.join(tempdir, 'contigs.fasta'), out_fa)
if os.path.isfile(out_fa):
with open(out_summary, 'w') as outh:
sequtils.assembly_stats(open(out_fa, 'rU'), outh)
if not keep_tmp:
sysutils.remove_tempdir(tempdir, 'assemble_spades', quiet, logfile)
return out_fa, out_summary
def summary_stats(dir_list=None, ph_list=None, quiet=False, logfile=None, debug=False, amplicons=False, outdir='.'):
# check for samtools
sysutils.check_dependency('samtools')
# check for dir_list (required)
if dir_list is not None:
f = open(dir_list, 'r')
filenames = f.read().splitlines()
else:
msg = 'no directory list given'
raise MissingRequiredArgument(msg)
# count number of samples
numsamps = 0
for f in filenames:
if len(f) > 0:
numsamps += 1
# count number of PH files
numph = 0
if ph_list is not None:
p = open(ph_list, 'r')
phnames = p.read().splitlines()
for f in phnames:
if len(f) > 0:
numph += 1
tsv_header = []
tsv_samps = []
with open(os.path.join(outdir, 'summary_stats.txt'), 'w') as outfile:
for i in range(numsamps): # for each sample
# set file names
bowtiefile = os.path.join(filenames[i], 'final_bt2.out')
trimfile = os.path.join(filenames[i], 'trimmomatic_summary.out')
bamfile = os.path.join(filenames[i], 'final.bam')
outidxstat = os.path.join(filenames[i], 'final.idxstat.txt')
finalfina = os.path.join(filenames[i], 'final.fna')
vcfzipped = os.path.join(filenames[i], 'final.vcf.gz')
vcfunzipped = os.path.join(filenames[i], 'final.vcf')
sampname = str(filenames[i])
num_cols = sampname.count('/') + 1
if i == 0: # if the first iteration, create tsv_header
for x in range(num_cols):
tsv_header += ['dir_%s' % str(x)]
tsv_header += ['RAW', 'CLEAN', 'ALN_RATE']
# output block 1
outfile.write("SAMPLE " + "%s:\n" % sampname)
outfile.write("\t Directory: %s\n" % str(os.path.abspath(filenames[i])))
raw = search_file(trimfile, "Input Read Pairs").split(' ')[3]
outfile.write("\t Number of raw read pairs: %s\n" % raw)
cleaned = search_file(bowtiefile, "reads;").split(' ')[0]
outfile.write("\t Number of cleaned read pairs: %s\n" % cleaned)
aln_rate = search_file(bowtiefile, "overall alignment rate").split(' ')[0]
outfile.write("\t Overall alignment rate: %s\n" % aln_rate)
# create tsv line
tsv_samp_temp = []
tsv_samp_temp += sampname.split('/')
tsv_samp_temp += [str(raw), str(cleaned), str(aln_rate)]
# index bam file with samtools
cmd0 = ["samtools index %s" % bamfile]
sysutils.command_runner([cmd0, ], 'summary_stats', quiet, logfile, debug)
# run idxstats with samtools
cmd1 = ["samtools idxstats %s > %s" % (bamfile, outidxstat)]
sysutils.command_runner([cmd1, ], 'summary_stats', quiet, logfile, debug)
# unzip vcf file
if os.path.isfile(vcfzipped):
cmd2 = ["gunzip %s" % vcfzipped]
sysutils.command_runner([cmd2, ], 'summary_stats', quiet, logfile, debug)
# if amplicon assembly
if amplicons is True:
all_amplicons = []
for record in SeqIO.parse(finalfina, 'fasta'):
reg_short = record.name.split('|')[5]
all_amplicons.append(str(reg_short))
# parse outidxstat and output
outfile.write("\t\t Amplicon %s:\n" % reg_short)
leng = search_file(outidxstat, reg_short).split('\t')[1]
outfile.write("\t\t\t Amplicon length: %s\n" % leng)
count = search_file(outidxstat, reg_short).split('\t')[2]
outfile.write("\t\t\t Amplicon read count: %s\n" % count)
# run depth with samtools for coverage
dep = os.path.join(filenames[i], 'final.depth.%s.txt' % reg_short)
cmd3 = ["samtools depth -r '%s' %s > %s" % (str(record.name), bamfile, dep)]
sysutils.command_runner([cmd3, ], 'summary_stats', quiet, logfile, debug)
# parse dep file from samtools
lines = 0
with open(dep) as depfile:
for line in depfile:
if len(line) > 0:
lines += 1
perc = (lines / int(leng)) * 100
# output coverage
outfile.write("\t\t\t Amplicon coverage: %s (%s percent)\n" % (lines, perc))
snps = parse_vcf_file(vcfunzipped, reg_short)
outfile.write("\t\t\t Number of SNPS: %s\n" % snps)
theta = float(snps) / float(leng)
outfile.write("\t\t\t Theta: %1.5f\n\n" % theta)
# add to tsv line
tsv_samp_temp += [str(leng), str(count), str(lines), str(perc), str(snps), str(theta)]
# add line to list for tsv
tsv_samps += [tsv_samp_temp]
# HAPLOTYPE FILES
if numph > 0:
outfile.write("\n\nHAPLPOTYPE SUMMARY STATISTICS\n\n")
ph_tsv_header = []
ph_tsv_samps = []
for i in range(numph): # for each PH directory
phfile = os.path.join(phnames[i], 'ph_summary.txt')
num_cols = phnames[i].count('/') + 1
if i == 0: # if the first iteration, create ph_tsv_header
for x in range(num_cols):
ph_tsv_header += ['dir_%s' % str(x)]
ph_tsv_header += ['PH_NUM_HAP', 'PH_HAP_DIVERSITY', 'PH_SEQ_LEN']
# output from parsing phfile
outfile.write("PH OUTPUT FILE %s:\n" % phfile)
num_hap = search_file(phfile, "PH_num_hap").split(' ')[1]
outfile.write("\t Number of haplotypes: %s\n" % num_hap)
div = search_file(phfile, "PH_hap_diversity").split(' ')[1]
outfile.write("\t Haplotype diversity: %s\n" % div)
seq_len = search_file(phfile, "PH_seq_len").split(' ')[1]
outfile.write("\t Sequence length: %s\n" % seq_len)
# create tsv line
ph_tsv_samp_temp = []
ph_tsv_samp_temp += phnames[i].split('/')
ph_tsv_samp_temp += [str(num_hap), str(div), str(seq_len)]
# add line to ph_tsv_samps
ph_tsv_samps += [ph_tsv_samp_temp]
# make summary_stats.tsv file
with open(os.path.join(outdir, 'summary_stats.tsv'), 'w') as outfile:
if amplicons is True:
for amp in all_amplicons:
tsv_header += ['%s_LEN' % amp, '%s_RC' % amp, '%s_COV_NUM' % amp,
'%s_COV_PERC' % amp, '%s_SNPS' % amp, '%s_THETA' % amp]
outfile.write(('\t').join(tsv_header) + '\n')
for samp in tsv_samps:
outfile.write(('\t').join(samp) + '\n')
# make PH_summary_stats.tsv file
if ph_list is not None:
with open(os.path.join(outdir, 'PH_summary_stats.tsv'), 'w') as outfile:
outfile.write(('\t').join(ph_tsv_header) + '\n')
for samp in ph_tsv_samps:
outfile.write(('\t').join(samp) + '\n')
# ending summary message
cmd3 = ['echo', 'Stage completed. Summary stats are located here: %s\n' % os.path.abspath('summary_stats.txt')]
if amplicons is True:
cmd3 += ['echo', 'Amplicons: %s\n' % (', ').join(all_amplicons)]
sysutils.command_runner([cmd3, ], 'summary_stats', quiet, logfile, debug)
def sample_reads(
fq1=None,
fq2=None,
fqU=None,
outdir='.',
nreads=None,
frac=None,
seed=None,
quiet=False,
logfile=None,
debug=False,
):
"""
Args:
fq1 (str): Path to fastq file with read 1
fq2 (str): Path to fastq file with read 2
fqU (str): Path to fastq file with unpaired reads
outdir (str): Path to output directory
nreads (int): Number of reads to sample
frac (float): Fraction of reads to sample
seed (int): Seed for random number generator
quiet (bool): Do not write output to console
logfile (file): Append console output to this file
debug (bool): Print commands but do not run
Returns:
out1 (str): Path to sampled fastq file with read 1
out2 (str): Path to sampled fastq file with read 2
outU (str): Path to sampled fastq file with unpaired reads
"""
# Check inputs
if fq1 is not None and fq2 is not None and fqU is None:
input_reads = "paired" # Paired end
elif fq1 is None and fq2 is None and fqU is not None:
input_reads = "single" # Single end
elif fq1 is not None and fq2 is not None and fqU is not None:
input_reads = "both"
else:
msg = "incorrect input reads; requires either "
msg += "(--fq1 AND --fq2) OR (--fqU) OR (--fq1 AND --fq2 AND --fqU)"
raise MissingRequiredArgument(msg)
# Check dependencies
sysutils.check_dependency('seqtk')
# Set seed
seed = seed if seed is not None else random.randrange(1, 1000)
sysutils.log_message('[--- sample_reads ---] Random seed = %d\n' % seed,
quiet, logfile)
# Set nreads/frac
if frac is not None:
if frac <= 0 or frac > 1:
raise sysutils.PipelineStepError('--frac must be > 0 and <= 1.')
frac_arg = '%f' % frac
else:
frac_arg = '%d' % nreads
cmds = None
if input_reads == 'single':
out1 = out2 = None
outU = os.path.join(outdir, 'sample_U.fastq')
cmds = [
[
'seqtk',
'sample',
'-s%d' % seed,
fqU,
frac_arg,
'>',
outU,
],
]
elif input_reads == 'paired':
out1 = os.path.join(outdir, 'sample_1.fastq')
out2 = os.path.join(outdir, 'sample_2.fastq')
outU = None
cmds = [
[
'seqtk',
'sample',
'-s%d' % seed,
fq1,
frac_arg,
'>',
out1,
],
[
'seqtk',
'sample',
'-s%d' % seed,
fq2,
frac_arg,
'>',
out2,
],
]
elif input_reads == 'both':
out1 = os.path.join(outdir, 'sample_1.fastq')
out2 = os.path.join(outdir, 'sample_2.fastq')
outU = os.path.join(outdir, 'sample_U.fastq')
cmds = [
[
'seqtk',
'sample',
'-s%d' % seed,
fq1,
frac_arg,
'>',
out1,
],
[
'seqtk',
'sample',
'-s%d' % seed,
fq2,
frac_arg,
'>',
out2,
],
[
'seqtk',
'sample',
'-s%d' % seed,
fqU,
frac_arg,
'>',
outU,
],
]
sysutils.command_runner(cmds, 'sample_reads', quiet, logfile, debug)
return out1, out2, outU
def cliquesnv(fq1=None,
fq2=None,
fqU=None,
ref_fa=None,
outdir='.',
jardir='.',
O22min=None,
O22minfreq=None,
printlog=None,
single=False,
merging=None,
fasta_format='extended4',
outputstart=None,
outputend=None,
keep_tmp=False,
quiet=False,
logfile=None,
debug=False,
ncpu=1):
# check if paired vs. single
if fq1 is None and fq2 is None and fqU is not None:
single = True
# check dependencies and required arguments
if fq1 is None and fq2 is None and fqU is None:
raise MissingRequiredArgument("No fastq files given.")
if single == False and (fq1 is None or fq2 is None):
raise MissingRequiredArgument("Either fq1 or fq2 missing.")
if ref_fa is None:
raise MissingRequiredArgument("Reference FASTA missing.")
sysutils.check_dependency('samtools')
sysutils.check_dependency('bwa')
if (os.path.isfile(os.path.join(jardir, "clique-snv.jar"))):
print("CliqueSNV JAR file found.")
else:
raise MissingRequiredArgument("No JAR file found.")
# Temporary directory
tempdir = sysutils.create_tempdir('clique_snv', None, quiet, logfile)
# Load reference fasta
refs = {s.id: s for s in SeqIO.parse(ref_fa, 'fasta')}
# Identify reconstruction regions
regions = []
for rname, s in refs.items():
regions.append(('cs%02d' % (len(regions) + 1), rname, 1, len(s)))
sysutils.log_message('[--- Haplotype Reconstruction Regions ---]\n', quiet,
logfile)
for iv in regions:
sysutils.log_message('%s -- %s:%d-%d\n' % iv, quiet, logfile)
if single == False: #paired end
# remove .1 and .2 from read names
fq1_c = os.path.join(tempdir, "fq1_corrected.fastq")
fq2_c = os.path.join(tempdir, "fq2_corrected.fastq")
cmd01 = ["cat %s | sed 's/\.1 / /' > %s" % (fq1, fq1_c)]
cmd02 = ["cat %s | sed 's/\.2 / /' > %s" % (fq2, fq2_c)]
sysutils.command_runner([cmd01, cmd02], 'clique_snv:setup', quiet,
logfile, debug)
# Create alignment for each REFERENCE in the reconstruction regions
alnmap = {}
for cs, rname, spos, epos in regions:
if rname not in alnmap:
# Create alignment
tmp_ref_fa = os.path.join(tempdir, 'ref.%d.fa' % len(alnmap))
tmp_sam = os.path.join(tempdir, 'aligned.%d.sam' % len(alnmap))
SeqIO.write(refs[rname], tmp_ref_fa, 'fasta')
cmd1 = [
'bwa',
'index',
tmp_ref_fa,
]
cmd2 = [
'bwa',
'mem',
tmp_ref_fa,
fq1_c,
fq2_c,
'|',
'samtools',
'view',
'-h',
'-F',
'12',
'>',
tmp_sam,
]
cmd3 = ['rm', '-f', '%s.*' % tmp_ref_fa]
sysutils.command_runner([cmd1, cmd2, cmd3], 'clique_snv:setup',
quiet, logfile, debug)
alnmap[rname] = (tmp_ref_fa, tmp_sam)
else: #single read
# Create alignment for each REFERENCE in the reconstruction regions
alnmap = {}
for cs, rname, spos, epos in regions:
if rname not in alnmap:
# Create alignment
tmp_ref_fa = os.path.join(tempdir, 'ref.%d.fa' % len(alnmap))
tmp_sam = os.path.join(tempdir, 'aligned.%d.sam' % len(alnmap))
SeqIO.write(refs[rname], tmp_ref_fa, 'fasta')
cmd1 = [
'bwa',
'index',
tmp_ref_fa,
]
cmd2 = [
'bwa',
'mem',
tmp_ref_fa,
fqU,
'|',
'samtools',
'view',
'-h',
'-F',
'12',
'>',
tmp_sam,
]
cmd3 = ['rm', '-f', '%s.*' % tmp_ref_fa]
sysutils.command_runner([cmd1, cmd2, cmd3], 'clique_snv:setup',
quiet, logfile, debug)
alnmap[rname] = (tmp_ref_fa, tmp_sam)
# Run CliqueSNV for each region
cmd4 = ['mkdir -p %s' % os.path.join(outdir, 'clique_snv')]
sysutils.command_runner([
cmd4,
],
stage='cliquesnv',
quiet=quiet,
logfile=logfile,
debug=debug)
i = 0 #index for filenames
for cs, rname, spos, epos in regions:
msg = "Reconstruction region %s:" % cs
msg += " %s:%d-%d\n" % (rname, spos, epos)
sysutils.log_message(msg, quiet, logfile)
# rename the cliquesnv number (cs##) to include region (now: cs##_reg)
cs = '%s_%s' % (cs, rname.split('|')[-2])
samfile = os.path.join(tempdir, 'aligned.%d.sam' % i)
method = 'snv-illumina'
cmd5 = [
'java -jar %s -m %s -in %s -threads %d -outDir %s -fdf %s' %
(os.path.join(jardir, 'clique-snv.jar'), method, samfile, ncpu,
tempdir, fasta_format)
]
if O22min is not None:
cmd5 += ['-t %f' % O22min]
if O22minfreq is not None:
cmd5 += ['-tf %f' % O22minfreq]
if printlog is not None:
cmd5 += ['-log']
if merging is not None:
cmd5 += ['-cm %s' % merging]
if outputstart is not None:
cmd5 += ['-os %d' % outputstart]
if outputend is not None:
cmd5 += ['-oe %d' % outputend]
sysutils.command_runner([
cmd5,
],
stage='clique_snv',
quiet=quiet,
logfile=logfile,
debug=debug)
# copy output file and delete tempdir
outname1 = 'aligned.%d.txt' % i
outname2 = 'aligned.%d.fasta' % i
os.makedirs(os.path.join(outdir, 'clique_snv/%s' % cs), exist_ok=True)
if os.path.exists(os.path.join(tempdir, '%s' % outname1)):
shutil.copy(
os.path.join(tempdir, '%s' % outname1),
os.path.join(outdir, 'clique_snv/%s/%s.txt' % (cs, cs)))
if os.path.exists(os.path.join(tempdir, '%s' % outname2)):
shutil.copy(
os.path.join(tempdir, '%s' % outname2),
os.path.join(outdir, 'clique_snv/%s/%s.fasta' % (cs, cs)))
# parse output file
with open(
os.path.join(outdir,
'clique_snv/%s/%s_summary.txt' % (cs, cs)),
'w') as sumfile, open(
os.path.join(outdir, 'clique_snv/%s/%s.txt' % (cs, cs)),
'r') as infile:
l = infile.readlines()
freqs = []
haps = []
tempnum = ''
for line in l:
if "SNV got" in line:
tempnum = line.split(' ')[2]
if "frequency" in line:
freqs += [float(line.split(' ')[2][:-2])]
if "haplotype=" in line:
haps += [line.split('=')[1][1:-2]]
sumfile.write('CliqueSNV_num_hap\t%s\n' % tempnum)
freq_sqrd = [x**2 for x in freqs]
freq_sqrd_sum = sum(freq_sqrd)
hap_div = ((old_div(7000, (7000 - 1))) * (1 - freq_sqrd_sum))
sumfile.write('CliqueSNV_hap_diversity\t%s\n' % hap_div)
sumfile.write('CliqueSNV_seq_len\t%s\n' % len(haps[0]))
with open(os.path.join(outdir, 'clique_snv/%s/%s.fasta' % (cs, cs)),
'r') as fastafile:
fastadata = fastafile.read().replace('aligned.%d' % i, rname)
with open(
os.path.join(outdir, 'clique_snv/%s/%s.fasta' % (cs, cs)),
'w') as newfastafile:
newfastafile.write(fastadata)
i += 1
if not keep_tmp:
sysutils.remove_tempdir(tempdir, 'clique_snv', quiet, logfile)
return