def matching_refseq(refseqs, k):
if k in refseqs:
return refseqs[k]
else:
if len(refseqs) > 1:
msg = 'No match for "%s" in reference sequences (%s)'
msg = msg % (k, ','.join(refseqs.keys()))
raise sysutils.PipelineStepError(msg)
return next(iter(refseqs.values()))
def assemble_denovo(**kwargs):
""" Wrapper function for denovo assembly
Arguments are passed to specific assembly functions depending on the
'assembler' argument
Args:
**kwargs: Arguments passed to specific assembly functions
Returns:
"""
if kwargs['assembler'] == 'spades':
return assemble_denovo_spades(**kwargs)
elif kwargs['assembler'] == 'trinity':
return assemble_denovo_trinity(**kwargs)
else:
raise sysutils.PipelineStepError('INVALID ASSEMBLER.')
def align_reads(
fq1=None,
fq2=None,
fqU=None,
ref_fa=None,
outdir='.',
bt2_preset='sensitive-local',
sample_id='sampleXX',
no_realign=False,
remove_duplicates=False,
encoding=None,
ncpu=1,
xmx=sysutils.get_java_heap_size(),
keep_tmp=False,
quiet=False,
logfile=None,
debug=False,
):
""" Pipeline step to align reads
Args:
fq1 (str): Path to fastq file with read 1
fq2 (str): Path to fastq file with read 2
fqU (str): Path to fastq file with unpaired reads
ref_fa (str): Path to reference fasta file
outdir (str): Path to output directory
bt2_preset (str): Bowtie2 preset to use for alignment
sample_id (str): Read group ID
no_realign (bool): Do not realign indels
remove_duplicates (bool): Remove duplicates from final alignment
encoding (str): Quality score encoding
ncpu (int): Number of CPUs to use
xmx (int): Maximum heap size for JVM in GB
keep_tmp (bool): Do not delete temporary directory
quiet (bool): Do not write output to console
logfile (file): Append console output to this file
debug (bool): Print commands but do not run
Returns:
out_aligned (str): Path to aligned BAM file
out_bt2 (str): Path to bowtie2 report
"""
# Check inputs
if fq1 is not None and fq2 is not None and fqU is None:
input_reads = "paired" # Paired end
elif fq1 is None and fq2 is None and fqU is not None:
input_reads = "single" # Single end
elif fq1 is not None and fq2 is not None and fqU is not None:
input_reads = "both"
else:
msg = "incorrect input reads; requires either "
msg += "(--fq1 AND --fq2) OR (--fqU) OR (--fq1 AND --fq2 AND --fqU)"
raise MissingRequiredArgument(msg)
if encoding is None:
if input_reads == 'single':
encoding = helpers.guess_encoding(fqU)
else:
encoding = helpers.guess_encoding(fq1)
# Check dependencies
sysutils.check_dependency('bowtie2')
sysutils.check_dependency('samtools')
sysutils.check_dependency('picard')
# Identify correct command for GATK
GATK_BIN = sysutils.determine_dependency_path(['gatk', 'gatk3'])
# Set JVM heap argument (for GATK)
JAVA_HEAP = '_JAVA_OPTIONS="-Xmx%dg"' % xmx
# Outputs
out_aligned = os.path.join(outdir, 'aligned.bam')
out_bt2 = os.path.join(outdir, 'aligned.bt2.out')
# Temporary directory
tempdir = sysutils.create_tempdir('align_reads', None, quiet, logfile)
# Copy and index initial reference
curref = os.path.join(tempdir, 'initial.fasta')
cmd1 = ['cp', ref_fa, curref]
cmd2 = ['samtools', 'faidx', curref]
cmd3 = [
'picard', 'CreateSequenceDictionary',
'R=%s' % curref,
'O=%s' % os.path.join(tempdir, 'initial.dict')
]
cmd4 = ['bowtie2-build', curref, os.path.join(tempdir, 'initial')]
sysutils.command_runner([cmd1, cmd2, cmd3, cmd4], 'align_reads:index',
quiet, logfile, debug)
# Align with bowtie2
cmd5 = [
'bowtie2',
'-p',
'%d' % ncpu,
'--phred33' if encoding == "Phred+33" else '--phred64',
'--no-unal',
'--rg-id',
sample_id,
'--rg',
'SM:%s' % sample_id,
'--rg',
'LB:1',
'--rg',
'PU:1',
'--rg',
'PL:illumina',
'--%s' % bt2_preset,
'-x',
'%s' % os.path.join(tempdir, 'initial'),
]
if input_reads in [
'paired',
'both',
]:
cmd5 += [
'-1',
fq1,
'-2',
fq2,
]
elif input_reads in [
'single',
'both',
]:
cmd5 += [
'-U',
fqU,
]
cmd5 += [
'-S',
os.path.join(tempdir, 'aligned.bt2.sam'),
]
cmd5 += [
'2>',
out_bt2,
]
try:
sysutils.command_runner([
cmd5,
], 'align_reads:bowtie2', quiet, logfile, debug)
except PipelineStepError as e:
if os.path.exists(out_bt2):
with open(out_bt2, 'r') as fh:
print('[--- bowtie2 stderr ---]\n%s' % fh.read(),
file=sys.stderr)
raise
cmd6 = [
'samtools',
'view',
'-u',
os.path.join(tempdir, 'aligned.bt2.sam'),
'|',
'samtools',
'sort',
'>',
os.path.join(tempdir, 'sorted.bam'),
]
cmd7 = [
'samtools',
'index',
os.path.join(tempdir, 'sorted.bam'),
]
sysutils.command_runner([
cmd6,
cmd7,
], 'align_reads:samsort', quiet, logfile, debug)
cur_bam = os.path.join(tempdir, 'sorted.bam')
if remove_duplicates:
sysutils.log_message('[--- Removing duplicates ---]', quiet, logfile)
else:
sysutils.log_message('[--- Marking duplicates ---]', quiet, logfile)
# MarkDuplicates
cmd8 = [
'picard',
'MarkDuplicates',
'CREATE_INDEX=true',
'USE_JDK_DEFLATER=true',
'USE_JDK_INFLATER=true',
'M=%s' % os.path.join(tempdir, 'rmdup.metrics.txt'),
'I=%s' % cur_bam,
'O=%s' % os.path.join(tempdir, 'rmdup.bam'),
]
if remove_duplicates:
cmd8 += [
'REMOVE_DUPLICATES=true',
]
sysutils.command_runner([
cmd8,
], 'align_reads:markdups', quiet, logfile, debug)
cur_bam = os.path.join(tempdir, 'rmdup.bam')
if no_realign:
print('[--- Skipping realignment ---]', file=sys.stderr)
else:
# RealignerTargetCreator
cmd9 = [
JAVA_HEAP,
GATK_BIN,
'-T',
'RealignerTargetCreator',
'-I',
cur_bam,
'-R',
curref,
'-o',
os.path.join(tempdir, 'tmp.intervals'),
]
# IndelRealigner
cmd10 = [
JAVA_HEAP, GATK_BIN, '-T', 'IndelRealigner', '--use_jdk_deflater',
'--use_jdk_inflater', '-maxReads', '1000000', '-dt', 'NONE', '-I',
cur_bam, '-R', curref, '-targetIntervals',
os.path.join(tempdir, 'tmp.intervals'), '-o',
os.path.join(tempdir, 'realign.bam')
]
sysutils.command_runner([
cmd9,
cmd10,
], 'align_reads:realign', quiet, logfile, debug)
cur_bam = os.path.join(tempdir, 'realign.bam')
# Check that cur_bam was created
if not os.path.exists(cur_bam):
msg = "BAM does not exist: %s" % cur_bam
raise sysutils.PipelineStepError(msg)
cmd11a = [
'rm',
'-f',
out_aligned,
]
cmd11b = [
'mv',
cur_bam,
out_aligned,
]
cmd11c = [
'samtools',
'index',
out_aligned,
]
sysutils.command_runner([
cmd11a,
cmd11b,
cmd11c,
], 'align_reads:copy', quiet, logfile, debug)
if not keep_tmp:
sysutils.remove_tempdir(tempdir, 'align_reads', quiet, logfile)
return out_aligned, out_bt2
def model_test(seqs=None,
outname='modeltest_results',
run_id=None,
data_type='nt',
partitions=None,
seed=None,
topology='ml',
utree=None,
force=None,
asc_bias=None,
frequencies=None,
het=None,
models=None,
schemes=None,
template=None,
ncpu=1,
quiet=False,
logfile=None,
debug=False,
outdir='.',
keep_tmp=False):
# check dependency
sysutils.check_dependency('modeltest-ng')
# check required input & input options
if seqs is None:
msg = "No alignment given"
raise sysutils.MissingRequiredArgument(msg)
if data_type not in ['nt', 'aa']:
raise sysutils.PipelineStepError("Data type not valid")
if topology not in [
'ml', 'mp', 'fixed-ml-jc', 'fixed-ml-gtr', 'fixed-mp', 'random',
'user'
]:
raise sysutils.PipelineStepError("Topology not valid")
# make tempdir
tempdir = sysutils.create_tempdir('model_test', None, quiet, logfile)
# add prefix
if run_id is not None:
outname = run_id + '_' + outname
# build command
cmd1 = [
'modeltest-ng -i %s' % seqs,
'-t %s' % topology,
'-o %s' % os.path.join(tempdir, outname),
'-p %d' % ncpu,
'-d %s' % data_type
]
if partitions is not None:
cmd1 += ['-q %s' % partitions]
if seed is not None:
cmd1 += ['-r %d' % seed]
if utree is not None:
cmd1 += ['-u %s' % utree]
if force is True:
cmd1 += ['--force']
if asc_bias is not None and asc_bias in [
'lewis', 'felsenstein', 'stamatakis'
]:
cmd1 += ['-a %s' % asc_bias]
elif asc_bias is not None:
raise sysutils.PipelineStepError("ASC bias correction not valid")
if frequencies is not None and frequencies in ['e', 'f']:
cmd1 += ['-f %s' % frequencies]
elif frequencies is not None:
raise sysutils.PipelineStepError("Frequencies not valid")
if het is not None and het in ['u', 'i', 'g', 'f']:
cmd1 += ['-h %s' % het]
elif het is not None:
raise sysutils.PipelineStepError("Rate heterogeneity not valid")
if models is not None:
with open(models, 'r') as f:
model_list = f.read().splitlines()
for m in model_list:
if data_type == 'nt' and m not in [
'JC', 'HKY', 'TrN', 'TPM1', 'TPM2', 'TPM3', 'TIM1', 'TIM2',
'TIM3', 'TVM', 'GTR'
]:
raise sysutils.PipelineStepError(
"At least one model is not valid")
elif data_type == 'aa' and m not in [
'DAYHOFF', 'LG', 'DCMUT', 'JTT', 'MTREV', 'WAG', 'RTREV',
'CPREV', 'VT', 'BLOSUM62', 'MTMAM', 'MTART', 'MTZOA',
'PMB', 'HIVB', 'HIVW', 'JTTDCMUT', 'FLU', 'SMTREV'
]:
raise sysutils.PipelineStepError(
"At least one model is not valid")
cmd1 += ['-m %s' % str(model_list)[1:-1]]
if schemes is not None and schemes in [3, 5, 7, 11, 203]:
cmd1 += ['-s %d' % schemes]
elif schemes is not None:
raise sysutils.PipelineStepError("Schemes not valid")
if template is not None and template in [
'raxml', 'phyml', 'mrbayes', 'paup'
]:
cmd1 += ['-T %s' % template]
elif template is not None:
raise sysutils.PipelineStepError("Template not valid")
# run command
try:
sysutils.command_runner([
cmd1,
], 'model_test', quiet, logfile, debug)
except sysutils.PipelineStepError as p:
if p.returncode == -6:
print("Warning: ignoring returncode -6")
else:
raise sysutils.PipelineStepError("Error in ModelTest-NG")
# copy output file and delete tempdir
if os.path.exists(os.path.join(tempdir, '%s.out' % outname)):
shutil.copy(os.path.join(tempdir, '%s.out' % outname),
os.path.abspath(outdir))
if not keep_tmp:
sysutils.remove_tempdir(tempdir, 'model_test', quiet, logfile)
# Parse .out file and write TSV summary file
criteria = []
bestmods = []
with open(os.path.join(outdir, '%s.out' % outname)) as f1:
for line in f1.read().splitlines():
if "Best model according to" in line:
criteria += line.split(' ')[-1:]
if "Model: " in line:
bestmods += line.split(' ')[-1:]
with open(os.path.join(outdir, '%s_summary.tsv' % outname), 'w') as f2:
f2.write('File\tCriteria\tBest Model\n')
for i in range(len(criteria)):
f2.write('%s\t%s\t%s\n' % (seqs, criteria[i], bestmods[i]))
# completion message
cmd2 = [
'echo',
'Stage completed. Output file is located here: %s\n' %
os.path.abspath(os.path.join(outdir, '%s.out' % outname)), 'echo',
'Summary TSV file is located here: %s\n' %
os.path.abspath(os.path.join(outdir, '%s_summary.tsv' % outname))
]
sysutils.command_runner([
cmd2,
], 'model_test', quiet, logfile, debug)
return
def build_tree_NG(seqs=None, in_type='FASTA', output_name='hp_tree', outdir='.',
treedir='hp_tree', model='GTR', bs_trees=None,
outgroup=None,branch_length=None, consense=None,rand_tree=None, pars_tree=None,
user_tree=None, search=None, search_1random=None, all=None,
constraint_tree=None,bsconverge=None, bs_msa=None,
bs_tree_cutoff=None,bs_metric=None, bootstrap=None, check=None,
log=None, loglh=None, redo=None,
terrace=None, seed=12345,version=None, quiet=False,
logfile=None, debug=False, ncpu=1,
keep_tmp=False):
sysutils.check_dependency('raxml-ng')
if version is True:
cmd2 = ['raxml-ng', '-v']
sysutils.command_runner([cmd2], 'build_tree_NG', quiet, logfile, debug)
return
if seqs is None:
msg = 'No alignment provided.'
raise sysutils.PipelineStepError(msg)
# Set Output Directory
output_dir = os.path.join(outdir, treedir)
cmd0 = ['mkdir -p %s' % output_dir]
sysutils.command_runner([cmd0], 'build_tree_NG', quiet, logfile, debug)
# fix seq names
if in_type=='FASTA':
check_name_compatibility(seqs,os.path.join(output_dir,'seqs_fixednames.fasta'),in_type)
elif in_type=='PHYLIP':
check_name_compatibility(seqs, os.path.join(output_dir, 'seqs_fixednames.phy'), in_type)
# Create temporary directory
tempdir = sysutils.create_tempdir('build_tree_NG', None, quiet, logfile)
# start raxml command
cmd1 = ['raxml-ng', '--prefix %s/%s' % (os.path.abspath(tempdir), output_name), '--threads %d' % ncpu, '--seed %d' % seed, '--model %s' % model]
if seqs is not None:
if in_type == 'FASTA':
cmd1 += ['--msa', '%s' % os.path.join(output_dir,'seqs_fixednames.fasta')]
elif in_type == 'PHYLIP':
cmd1 += ['--msa', '%s' % os.path.join(output_dir, 'seqs_fixednames.phy')]
if branch_length is not None:
cmd1 += ['--brlen', '%s' % branch_length]
if consense is not None:
cmd1 += ['--consense', '%s' % consense]
if pars_tree is not None and rand_tree is None:
cmd1 += ['--tree pars{%d}' % pars_tree]
if pars_tree is None and rand_tree is not None:
cmd1 += ['--tree rand{%d}' % rand_tree]
if pars_tree is not None and rand_tree is not None:
cmd1 += ['--tree pars{%d},rand{%d}' % (pars_tree, rand_tree)]
if user_tree is not None:
cmd1 += ['--tree', '%s' % os.path.abspath(user_tree)]
if search is True:
cmd1 += ['--search']
if search_1random is True:
cmd1 += ['--search1']
if all is True:
cmd1 += ['--all']
if constraint_tree is not None:
cmd1 += ['--tree-constraint', '%s' % os.path.abspath(constraint_tree)]
if outgroup is not None:
cmd1 += ['--outgroup', '%s' % outgroup]
if bsconverge is True:
cmd1 += ['--bsconverge']
if bs_msa is True:
cmd1 += ['--bsmsa']
if bs_trees is not None:
cmd1 += ['--bs-trees %s' % bs_trees]
if bs_tree_cutoff is not None:
cmd1 += ['--bs-cutoff', '%f' % bs_tree_cutoff]
if bs_metric is not None:
cmd1 += ['--bs-metric', '%s' % bs_metric]
if bootstrap is True:
cmd1 += ['--bootstrap']
if check is True:
cmd1 += ['--check']
if log is not None:
cmd1 += ['--log', '%s' % log]
if loglh is True:
cmd1 += ['--loglh']
if terrace is True:
cmd1 += ['--terrace']
if redo is not None:
cmd1 += ['--redo']
sysutils.command_runner([cmd1, ], 'build_tree_NG', quiet, logfile, debug)
# copy files from tmpdir to output directory (note - took some out here)
if os.path.exists(os.path.join(tempdir, '%s.raxml.bestTree' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.bestTree' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.bestPartitionTrees' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.bestPartitionTrees' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.bestModel' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.bestModel' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.bootstraps' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.bootstraps' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.bootstrapMSA.<REP>.phy' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.bootstrapMSA.<REP>.phy' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.ckp' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.ckp' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.consensusTree' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.consensusTree' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.log' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.log' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.mlTrees' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.mlTrees' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.startTree' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.startTree' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.support' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.support' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.terrace' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.terrace' % output_name), os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, '%s.raxml.terraceNewick' % output_name)):
shutil.copy(os.path.join(tempdir, '%s.raxml.terraceNewick' % output_name), os.path.abspath(output_dir))
if not keep_tmp:
sysutils.remove_tempdir(tempdir, 'build_tree_NG', quiet, logfile)
cmd6 = ['echo', 'Stage completed. Output files are located here: %s\n' % os.path.abspath(output_dir)]
sysutils.command_runner([cmd6, ], 'build_tree_NG', quiet, logfile, debug)
def ec_reads(
fq1=None,
fq2=None,
fqU=None,
outdir='.',
ncpu=1,
keep_tmp=False,
quiet=False,
logfile=None,
debug=False,
):
""" Pipeline step to error-correct reads using spades
Args:
fq1 (str): Path to fastq file with read 1
fq2 (str): Path to fastq file with read 2
fqU (str): Path to fastq file with unpaired reads
outdir (str): Path to output directory
ncpu (int): Number of CPUs to use
keep_tmp (bool): Do not delete temporary directory
quiet (bool): Do not write output to console
logfile (file): Append console output to this file
debug (bool): Print commands but do not run
Returns:
out1 (str): Path to corrected fastq file with read 1
out2 (str): Path to corrected fastq file with read 2
outU (str): Path to corrected fastq file with unpaired reads
"""
# Check inputs
if fq1 is not None and fq2 is not None and fqU is None:
input_reads = "paired" # Paired end
elif fq1 is None and fq2 is None and fqU is not None:
input_reads = "single" # Single end
elif fq1 is not None and fq2 is not None and fqU is not None:
input_reads = "both"
else:
msg = "incorrect input reads; requires either "
msg += "(--fq1 AND --fq2) OR (--fqU) OR (--fq1 AND --fq2 AND --fqU)"
raise MissingRequiredArgument(msg)
# Check dependencies
sysutils.check_dependency('spades.py')
# Outputs
out1 = os.path.join(outdir, 'corrected_1.fastq')
out2 = os.path.join(outdir, 'corrected_2.fastq')
outU = os.path.join(outdir, 'corrected_U.fastq')
# Temporary directory
tempdir = sysutils.create_tempdir('ec_reads', None, quiet, logfile)
# spades command
cmd1 = [
'spades.py',
'-o',
tempdir,
'-t',
'%d' % ncpu,
'--only-error-correction',
]
if input_reads in [
'paired',
'both',
]:
cmd1 += [
'-1',
os.path.abspath(fq1),
'-2',
os.path.abspath(fq2),
]
if input_reads in [
'single',
'both',
]:
cmd1 += [
'-s',
os.path.abspath(fqU),
]
sysutils.command_runner([
cmd1,
], 'ec_reads', quiet, logfile, debug)
# Copy files
yaml_file = os.path.join(tempdir, 'corrected/corrected.yaml')
if not os.path.exists(yaml_file):
sysutils.PipelineStepError("YAML file %s not found" % yaml_file)
with open(yaml_file, 'rU') as fh:
d = yaml.load(fh, Loader=yaml.FullLoader)[0]
cmds = []
if 'left reads' in d:
cmds.append([
'gunzip',
'-c',
] + sorted(d['left reads']) + ['>', out1])
if 'right reads' in d:
cmds.append([
'gunzip',
'-c',
] + sorted(d['right reads']) + ['>', out2])
if 'single reads' in d:
cmds.append([
'gunzip',
'-c',
] + sorted(d['single reads']) + ['>', outU])
sysutils.command_runner(cmds, 'ec_reads', quiet, logfile, debug)
if not keep_tmp:
sysutils.remove_tempdir(tempdir, 'ec_reads', quiet, logfile)
return out1, out2, outU
def run_mafft(inputseqs=None,
out_align="alignment.fasta",
auto=None,
algo=None,
sixmerpair=None,
globalpair=None,
localpair=None,
genafpair=None,
fastapair=None,
weighti=None,
retree=None,
maxiterate=None,
noscore=None,
memsave=None,
parttree=None,
dpparttree=None,
fastaparttree=None,
partsize=None,
groupsize=None,
lop=None,
lep=None,
lexp=None,
LOP=None,
LEXP=None,
bl=None,
jtt=None,
tm=None,
aamatrix=None,
fmodel=None,
clustalout=None,
inputorder=None,
reorder=None,
treeout=None,
quiet_mafft=None,
nuc=None,
amino=None,
quiet=False,
logfile=None,
debug=False,
ncpu=1,
msadir='.',
phylipout=None):
### function to run MAFFT ###
sysutils.check_dependency('mafft')
## create MAFFT command using input options
if algo is None:
cmd1 = [
'mafft',
'--thread',
'%d' % ncpu,
]
else:
if algo not in [
'linsi', 'ginsi', 'einsi', 'fftnsi', 'fftns', 'nwns', 'nwnsi'
]:
msg = 'Algorithm not in MAFFT'
raise sysutils.PipelineStepError(msg)
else:
cmd1 = ['%s' % algo]
if clustalout is True:
cmd1 += ['--clustalout']
if inputorder is True:
cmd1 += ['--inputourder']
if reorder is True:
cmd1 += ['--reorder']
if treeout is True:
cmd1 += ['--treeout']
if quiet_mafft is True:
cmd1 += ['--quiet']
if nuc is True:
cmd1 += ['--nuc']
if amino is True:
cmd1 += ['--amino']
### algorithm options
if auto is True:
cmd1 += ['--auto']
if sixmerpair is True:
cmd1 += ['--6merpair']
if globalpair is True:
cmd1 += ['--globalpair']
if localpair is True:
cmd1 += ['--localpair']
if genafpair is True:
cmd1 += ['--genafpair']
if fastapair is True:
cmd1 += ['--fastapair']
if weighti is not None:
cmd1 += ['--weighti', '%f' % weighti]
if retree is not None:
cmd1 += ['--retree', '%d' % retree]
if maxiterate is not None:
cmd1 += ['--maxiterate', '%d' % maxiterate]
if noscore is True:
cmd1 += ['--noscore']
if memsave is True:
cmd1 += ['--memsave']
if parttree is True:
cmd1 += ['--parttree']
if dpparttree is True:
cmd1 += ['--dpparttree']
if fastaparttree is True:
cmd1 += ['--fastaparttree']
if partsize is not None:
cmd1 += ['--partsize', '%d' % partsize]
if groupsize is not None:
cmd1 += ['--groupsize', '%d' % groupsize]
### parameters
if lop is not None:
cmd1 += ['--lop', '%f' % lop]
if lep is not None:
cmd1 += ['--lep', '%f' % lep]
if lexp is not None:
cmd1 += ['--lexp', '%f' % lexp]
if LOP is not None:
cmd1 += ['--LOP', '%f' % LOP]
if LEXP is not None:
cmd1 += ['--LEXP', '%f' % LEXP]
if bl is not None:
cmd1 += ['--bl', '%d' % bl]
if jtt is not None:
cmd1 += ['--jtt', '%d' % jtt]
if tm is not None:
cmd1 += ['--tm', '%d' % tm]
if aamatrix is not None:
cmd1 += ['--aamatrix', '%s' % aamatrix]
if fmodel is True:
cmd1 += ['--fmodel']
# Outputs
outName = os.path.join(msadir, '%s' % os.path.basename(out_align))
## create command
cmd1 += ['%s' % inputseqs, '>', '%s' % outName]
## run MAFFT command
sysutils.command_runner([
cmd1,
], 'multiple_align', quiet, logfile, debug)
if phylipout is True:
phyout = outName[:-6] + '.phy'
SeqIO.convert(
outName, 'fasta', phyout,
'phylip-relaxed') # relaxed allows for long sequence names
cmd2 = ['echo', 'Output converted to PHYLIP format from FASTA format.']
sysutils.command_runner([
cmd2,
], 'multiple_align', quiet, logfile, debug)
if clustalout is True:
clustout = outName[:-6] + '.aln'
cmd3 = ['mv', outName, clustout]
sysutils.command_runner([
cmd3,
], 'multiple_align', quiet, logfile, debug)
cmd4 = ['echo', 'Alignment output is in CLUSTAL format.']
sysutils.command_runner([
cmd4,
], 'multiple_align', quiet, logfile, debug)
return
def join_reads(
fq1=None,
fq2=None,
outdir=".",
min_overlap=None,
max_overlap=None,
allow_outies=None,
encoding=None,
ncpu=1,
keep_tmp=False,
quiet=False,
logfile=None,
debug=False,
):
""" Pipeline step to join paired-end reads
Args:
fq1 (str): Path to fastq file with read 1
fq2 (str): Path to fastq file with read 2
outdir (str): Path to output directory
min_overlap (int): The minimum required overlap length
max_overlap (int): Maximum overlap length
allow_outies (bool): Try combining "outie" reads
encoding (str): Quality score encoding
ncpu (int): Number of CPUs to use
keep_tmp (bool): Do not delete temporary directory
quiet (bool): Do not write output to console
logfile (file): Append console output to this file
debug (bool): Print commands but do not run
Returns:
out1 (str): Path to fastq file with unjoined read 1
out2 (str): Path to fastq file with unjoined read 2
outU (str): Path to fastq file with joined reads
"""
# Check inputs
if fq1 is not None and fq2 is not None:
pass # Both are present
else:
msg = "Incorrect combination of reads: fq1=%s fq2=%s" % (fq1, fq2)
raise sysutils.PipelineStepError(msg)
# Check for executable
sysutils.check_dependency('flash')
# Get encoding
if encoding is None:
encoding = helpers.guess_encoding(fq1)
# Outputs
outU = os.path.join(outdir, 'joined.fastq')
out1 = os.path.join(outdir, 'notjoined_1.fastq')
out2 = os.path.join(outdir, 'notjoined_2.fastq')
# Temporary directory
tempdir = sysutils.create_tempdir('join_reads', None, quiet, logfile)
# Flash command
cmd1 = [
'flash',
'-t',
'%d' % ncpu,
'-d',
tempdir,
]
if encoding != "Phred+33":
cmd1 += ['-p', '64']
if min_overlap is not None:
cmd1 += ['-m', '%d' % min_overlap]
if max_overlap is not None:
cmd1 += ['-M', '%d' % max_overlap]
if allow_outies is True:
cmd1 += ['-O']
cmd1 += [fq1, fq2]
cmd2 = [
'mv',
os.path.join(tempdir, 'out.extendedFrags.fastq'),
outU,
]
cmd3 = [
'mv',
os.path.join(tempdir, 'out.notCombined_1.fastq'),
out1,
]
cmd4 = [
'mv',
os.path.join(tempdir, 'out.notCombined_2.fastq'),
out2,
]
sysutils.command_runner([
cmd1,
cmd2,
cmd3,
cmd4,
], 'join_reads', quiet, logfile, debug)
if not keep_tmp:
sysutils.remove_tempdir(tempdir, 'join_reads', quiet, logfile)
return out1, out2, outU
def vcf_to_consensus(
vcf=None,
outdir='.',
sampidx=0,
min_dp=5,
major=0.5,
minor=0.2,
keep_tmp=False,
quiet=False,
logfile=None,
):
""" Pipeline step to create consensus sequence from VCF
Args:
vcf (str): Path to variant calls (VCF)
outdir (str): Path to output directory
sampidx (int): Index for sample if multi-sample VCF
min_dp (int): Minimum depth to call site
major (float): Allele fraction to make unambiguous call
minor (float): Allele fraction to make ambiguous call
keep_tmp (bool): Do not delete temporary directory
quiet (bool): Do not write output to console
logfile (file): Append console output to this file
debug (bool): Print commands but do not run
Returns:
"""
# Check inputs
if vcf is None:
raise sysutils.PipelineStepError('VCF file is required')
# Outputs
out_fasta = os.path.join(outdir, 'consensus.fna')
sysutils.log_message('[--- vcf_to_consensus ---]\n', quiet, logfile)
sysutils.log_message('VCF: %s\n' % vcf, quiet, logfile)
# Parse VCF
chroms = []
samples = []
if os.path.splitext(vcf)[1] == '.gz':
lines = (l.decode('utf-8').strip('\n') for l in gzip.open(vcf, 'rb'))
else:
lines = (l.strip('\n') for l in open(vcf, 'r'))
# Parse headers
for l in lines:
if l.startswith('##'):
m = re.match('##contig=<ID=(\S+),length=(\d+)>', l)
if m:
chroms.append((m.group(1), int(m.group(2))))
else:
assert l.startswith('#')
cols = l.strip('#').split('\t')
samples = cols[9:]
break
if len(samples) <= sampidx:
msg = 'Sample index %d does not exist. Samples: %s' % (sampidx,
str(samples))
raise sysutils.PipelineStepError(msg)
chrom_ordered = [_[0] for _ in chroms]
chroms = dict(chroms)
newseqs = dict((c, ['.'] * chroms[c]) for c in list(chroms.keys()))
imputed = dict((c, [''] * chroms[c]) for c in list(chroms.keys()))
for l in lines:
chrom, start, stop, RA, AA, info, svals = parse_vcf_sample(l, sampidx)
gt = call_gt(RA, AA, svals, min_dp, major, minor)
if gt is None:
imputed[chrom][start - 1] = RA[0].lower()
else:
if len(gt) == 1:
newseqs[chrom][start - 1] = gt[0]
imputed[chrom][start - 1] = gt[0]
else:
if all(len(_) == 1 for _ in gt):
newseqs[chrom][start - 1] = sequtils.get_ambig(gt)
imputed[chrom][start - 1] = sequtils.get_ambig(gt)
else:
newseqs[chrom][start - 1] = ''.join(gt[0])
imputed[chrom][start - 1] = ''.join(gt[0])
# newseqs = imputed
sysutils.log_message('Output FASTA: %s\n' % out_fasta, quiet, logfile)
with open(out_fasta, 'w') as outh:
for chrom in chrom_ordered:
new_seqid = sequtils.update_seq_id(chrom, samples[sampidx])
new_seq = ''.join(newseqs[chrom]).replace('.', 'n')
m = re.match('^(?P<pre>n*)(?P<seq>[^n].+[^n])?(?P<suf>n*)$',
new_seq)
if m.group('seq') is None:
msg = u'%s\tFAIL\t%d\t%s\n' % (new_seqid, 0, u"👎🏼")
# Don't output sequence if not present
else:
msg = u'%s\tPASS\t%d\t%s\n' % (new_seqid, len(
m.group('seq')), u"👍🏼")
print('>%s SM:%s' % (new_seqid, samples[sampidx]), file=outh)
print(sequtils.wrap(new_seq), file=outh)
sysutils.log_message(msg, quiet, logfile)
return out_fasta
def sample_reads(
fq1=None,
fq2=None,
fqU=None,
outdir='.',
nreads=None,
frac=None,
seed=None,
quiet=False,
logfile=None,
debug=False,
):
"""
Args:
fq1 (str): Path to fastq file with read 1
fq2 (str): Path to fastq file with read 2
fqU (str): Path to fastq file with unpaired reads
outdir (str): Path to output directory
nreads (int): Number of reads to sample
frac (float): Fraction of reads to sample
seed (int): Seed for random number generator
quiet (bool): Do not write output to console
logfile (file): Append console output to this file
debug (bool): Print commands but do not run
Returns:
out1 (str): Path to sampled fastq file with read 1
out2 (str): Path to sampled fastq file with read 2
outU (str): Path to sampled fastq file with unpaired reads
"""
# Check inputs
if fq1 is not None and fq2 is not None and fqU is None:
input_reads = "paired" # Paired end
elif fq1 is None and fq2 is None and fqU is not None:
input_reads = "single" # Single end
elif fq1 is not None and fq2 is not None and fqU is not None:
input_reads = "both"
else:
msg = "incorrect input reads; requires either "
msg += "(--fq1 AND --fq2) OR (--fqU) OR (--fq1 AND --fq2 AND --fqU)"
raise MissingRequiredArgument(msg)
# Check dependencies
sysutils.check_dependency('seqtk')
# Set seed
seed = seed if seed is not None else random.randrange(1, 1000)
sysutils.log_message('[--- sample_reads ---] Random seed = %d\n' % seed,
quiet, logfile)
# Set nreads/frac
if frac is not None:
if frac <= 0 or frac > 1:
raise sysutils.PipelineStepError('--frac must be > 0 and <= 1.')
frac_arg = '%f' % frac
else:
frac_arg = '%d' % nreads
cmds = None
if input_reads == 'single':
out1 = out2 = None
outU = os.path.join(outdir, 'sample_U.fastq')
cmds = [
[
'seqtk',
'sample',
'-s%d' % seed,
fqU,
frac_arg,
'>',
outU,
],
]
elif input_reads == 'paired':
out1 = os.path.join(outdir, 'sample_1.fastq')
out2 = os.path.join(outdir, 'sample_2.fastq')
outU = None
cmds = [
[
'seqtk',
'sample',
'-s%d' % seed,
fq1,
frac_arg,
'>',
out1,
],
[
'seqtk',
'sample',
'-s%d' % seed,
fq2,
frac_arg,
'>',
out2,
],
]
elif input_reads == 'both':
out1 = os.path.join(outdir, 'sample_1.fastq')
out2 = os.path.join(outdir, 'sample_2.fastq')
outU = os.path.join(outdir, 'sample_U.fastq')
cmds = [
[
'seqtk',
'sample',
'-s%d' % seed,
fq1,
frac_arg,
'>',
out1,
],
[
'seqtk',
'sample',
'-s%d' % seed,
fq2,
frac_arg,
'>',
out2,
],
[
'seqtk',
'sample',
'-s%d' % seed,
fqU,
frac_arg,
'>',
outU,
],
]
sysutils.command_runner(cmds, 'sample_reads', quiet, logfile, debug)
return out1, out2, outU
def build_tree(seqs=None,
data_type='NUC',
run_full_analysis=None,
output_name='build_tree.tre',
outdir='.',
treedir='hp_build_tree',
model='GTRGAMMAIX',
outgroup=None,
parsimony_seed=1234,
wgtFile=None,
secsub=None,
bootstrap=None,
bootstrap_threshold=None,
numCat=None,
rand_starting_tree=None,
convergence_criterion=None,
likelihoodEpsilon=None,
excludeFileName=None,
algo_option=None,
cat_model=None,
groupingFile=None,
placementThreshold=None,
disable_pattern_compression=None,
InitialRearrangement=None,
posteriori=None,
print_intermediate_trees=None,
majorityrule=None,
print_branch_length=None,
ICTCmetrics=None,
partition_branch_length=None,
disable_check=None,
AAmodel=None,
multiplemodelFile=None,
binarytree=None,
BinaryParameterFile=None,
SecondaryStructure=None,
UserStartingTree=None,
median_GAMMA=None,
version_info=None,
rate_heterogeneity=None,
window=None,
RapidBootstrapNumSeed=None,
random_addition=None,
starting_tree=None,
quartetGroupingFileName=None,
multipleTreeFile=None,
NumberofRuns=None,
mesquite=None,
silent=None,
noseqcheck=None,
nobfgs=None,
epaPlaceNum=None,
epaProbThreshold=None,
epaLikelihood=None,
HKY85=None,
BootstrapPerm=None,
quiet=False,
logfile=None,
debug=False,
keep_tmp=False,
option_help=None):
# Check dependencies
sysutils.check_dependency('raxmlHPC')
cmd1 = []
# check for required input
if option_help is True:
cmd4 = ['raxmlHPC', '-h']
sysutils.command_runner([cmd4], 'build_tree', quiet, logfile, debug)
return
if version_info is True:
cmd5 = ['raxmlHPC', '-v']
sysutils.command_runner([cmd5], 'build_tree', quiet, logfile, debug)
if seqs is None and option_help is None:
msg = 'No alignment provided'
raise sysutils.PipelineStepError(msg)
# check model compatibility
if data_type is not 'AA' and 'PROT' in model:
msg = 'Protein model given for non-amino acid data'
raise sysutils.PipelineStepError(msg)
if data_type is not 'MULTI' and 'MULTI' in model:
msg = 'Multi-state model given for non-multi-state data'
raise sysutils.PipelineStepError(msg)
if data_type is not 'BIN' and 'BIN' in model:
msg = 'Binary model given for non-binary data'
raise sysutils.PipelineStepError(msg)
if data_type is not 'NUC':
if data_type not in model:
msg = 'model and data type not compatible'
raise sysutils.PipelineStepError(msg)
# Set Output Directory
output_dir = os.path.join(outdir, treedir)
cmd0 = ['mkdir -p %s' % output_dir]
sysutils.command_runner([cmd0], 'build_tree', quiet, logfile, debug)
# Temporary directory
tempdir = sysutils.create_tempdir('build_tree', None, quiet, logfile)
if run_full_analysis is True:
# generate seeds
seed1 = random.randint(10000, 99999)
seed2 = random.randint(10000, 99999)
cmd1 = [
'echo',
'Using parsimony seed %s and bootstrap seed %s' % (seed1, seed2)
]
sysutils.command_runner([cmd1], 'build_tree', quiet, logfile, debug)
# run raxml
cmd2 = [
'raxmlHPC',
'-w %s' % os.path.abspath(tempdir), '-f a',
'-p %d' % seed1,
'-x %d' % seed2, '-# 100',
'-m %s' % model,
'-s %s' % os.path.abspath(seqs),
'-n %s' % output_name
]
sysutils.command_runner([cmd2], 'build_tree', quiet, logfile, debug)
else:
# start raxml command
cmd1 = [
'raxmlHPC',
'-w %s' % os.path.abspath(tempdir),
'-p %d' % parsimony_seed,
'-m %s' % model
]
if outgroup is not None:
cmd1 += ['-o', '%s' % outgroup]
if wgtFile is not None:
cmd1 += ['-a', '%s' % os.path.join('.', wgtFile)]
if secsub is not None and SecondaryStructure is not None:
cmd1 += ['-A', '%s' % secsub]
cmd1 += ['-S', '%s' % os.path.join('.', SecondaryStructure)]
elif secsub is not None and SecondaryStructure is None:
msg = 'Need to specify a file defining the secondary structure via the S option'
raise sysutils.PipelineStepError(msg)
if bootstrap is not None:
cmd1 += ['-b', '%d' % bootstrap]
if bootstrap_threshold is not None:
cmd1 += ['-B', '%f' % bootstrap_threshold]
if numCat is not None:
cmd1 += ['-c', '%d' % numCat]
if rand_starting_tree is True:
cmd1 += ['-d']
if convergence_criterion is True:
cmd1 += ['-D']
if likelihoodEpsilon is not None:
cmd1 += ['-e', '%f' % likelihoodEpsilon]
if excludeFileName is not None:
cmd1 += ['-E', '%s' % os.path.join('.', excludeFileName)]
if algo_option is not None and algo_option in [
'a', 'A', 'b', 'B', 'c', 'C', 'd', 'D', 'e', 'E', 'F', 'g',
'G', 'h', 'H', 'i', 'I', 'j', 'J', 'k', 'm', 'n', 'N', 'o',
'p', 'q', 'r', 'R', 's', 'S', 't', 'T', 'U', 'v', 'V', 'w',
'W', 'x', 'y'
]:
cmd1 += ['-f', '%s' % algo_option]
if cat_model is True:
cmd1 += ['-F']
if groupingFile is not None:
cmd1 += ['-g', '%s' % os.path.join('.', groupingFile)]
if placementThreshold is not None:
cmd1 += ['-G', '%f' % placementThreshold]
if disable_pattern_compression is True:
cmd1 += ['-H']
if InitialRearrangement is not None:
cmd1 += ['-i', '%d' % InitialRearrangement]
if posteriori is not None:
cmd1 += ['-I', '%s' % posteriori]
if print_intermediate_trees is True:
cmd1 += ['-j']
if (majorityrule is not None) and (multipleTreeFile is not None):
cmd1 += ['-J', '%s' % majorityrule]
cmd1 += ['-z', '%s' % os.path.join('.', multipleTreeFile)]
elif majorityrule is not None and multipleTreeFile is None:
msg = 'Need to provide a tree file containing several UNROOTED trees via the z option'
raise sysutils.PipelineStepError(msg)
if print_branch_length is True:
cmd1 += ['-k']
if ICTCmetrics is not None:
cmd1 += ['-L', '%s' % ICTCmetrics]
if partition_branch_length is True:
cmd1 += ['-M']
if disable_check is True:
cmd1 += ['-O']
if AAmodel is not None:
cmd1 += ['-P', '%s' % os.path.join('.', AAmodel)]
if multiplemodelFile is not None:
cmd1 += ['-q', '%s' % os.path.join('.', multiplemodelFile)]
if binarytree is not None:
cmd1 += ['-r', '%s' % os.path.join('.', binarytree)]
if BinaryParameterFile is not None:
cmd1 += ['-R', '%s' % os.path.join('.', BinaryParameterFile)]
if SecondaryStructure is not None:
cmd1 += ['-S', '%s' % os.path.join('.', SecondaryStructure)]
if UserStartingTree is not None:
cmd1 += ['-t', '%s' % os.path.join('.', UserStartingTree)]
if median_GAMMA is True:
cmd1 += ['-u']
if rate_heterogeneity is True:
cmd1 += ['-V']
if window is not None:
cmd1 += ['-W', '%d' % window]
if RapidBootstrapNumSeed is not None:
cmd1 += ['-x', '%d' % RapidBootstrapNumSeed]
if random_addition is True:
cmd1 += ['-X']
if starting_tree is True:
cmd1 += ['-y']
if quartetGroupingFileName is not None:
cmd1 += ['-Y', '%s' % os.path.join('.', quartetGroupingFileName)]
if multipleTreeFile is not None:
cmd1 += ['-z', '%s' % os.path.join('.', multipleTreeFile)]
if NumberofRuns is not None:
cmd1 += ['-N', '%d' % NumberofRuns]
if mesquite is True:
cmd1 += ['--mesquite']
if silent is True:
cmd1 += ['--silent']
if noseqcheck is True:
cmd1 += ['--no-seq-check']
if nobfgs is True:
cmd1 += ['--no-bfgs']
if epaPlaceNum is not None:
cmd1 += ['epakeepplacements=%d' % epaPlaceNum]
if epaProbThreshold is not None:
cmd1 += ['epaprobthreshold=%f' % epaProbThreshold]
if epaLikelihood is not None:
cmd1 += ['epaaccumulatedthreshold=%f' % epaLikelihood]
if HKY85 is True:
cmd1 += ['--HKY85']
if BootstrapPerm is not None:
cmd1 += ['[bootstopperms=%s' % BootstrapPerm]
if option_help is True:
cmd1 += ['-h']
cmd1 += ['-s', '%s' % os.path.abspath(seqs), '-n', '%s' % output_name]
sysutils.command_runner([
cmd1,
], 'build_tree', quiet, logfile, debug)
# copy files from tmpdir to output directory
if os.path.exists(os.path.join(tempdir,
'RAxML_bestTree.%s' % output_name)):
shutil.copy(os.path.join(tempdir, 'RAxML_bestTree.%s' % output_name),
os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, 'RAxML_info.%s' % output_name)):
shutil.copy(os.path.join(tempdir, 'RAxML_info.%s' % output_name),
os.path.abspath(output_dir))
if os.path.exists(
os.path.join(tempdir, 'RAxML_perSiteLLs.%s' % output_name)):
shutil.copy(os.path.join(tempdir, 'RAxML_perSiteLLs.%s' % output_name),
os.path.abspath(output_dir))
if os.path.exists(
os.path.join(tempdir,
'RAxML_bipartitionFrequencies.%s' % output_name)):
shutil.copy(
os.path.join(tempdir,
'RAxML_bipartitionFrequencies.%s' % output_name),
os.path.abspath(output_dir))
if os.path.exists(
os.path.join(tempdir,
'RAxML_bipartitionsBranchLabels.%s' % output_name)):
shutil.copy(
os.path.join(tempdir,
'RAxML_bipartitionsBranchLabels.%s' % output_name),
os.path.abspath(output_dir))
if os.path.exists(
os.path.join(tempdir, 'RAxML_bipartitions.%s' % output_name)):
shutil.copy(
os.path.join(tempdir, 'RAxML_bipartitions.%s' % output_name),
os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir,
'RAxML_bootstrap.%s' % output_name)):
shutil.copy(os.path.join(tempdir, 'RAxML_bootstrap.%s' % output_name),
os.path.abspath(output_dir))
if os.path.exists(
os.path.join(tempdir, 'RAxML_checkpoint.%s' % output_name)):
shutil.copy(os.path.join(tempdir, 'RAxML_checkpoint.%s' % output_name),
os.path.abspath(output_dir))
if os.path.exists(
os.path.join(tempdir, 'RAxML_randomTree.%s' % output_name)):
shutil.copy(os.path.join(tempdir, 'RAxML_randomTree.%s' % output_name),
os.path.abspath(output_dir))
if os.path.exists(
os.path.join(tempdir, 'RAxML_parsimonyTree.%s' % output_name)):
shutil.copy(
os.path.join(tempdir, 'RAxML_parsimonyTree.%s' % output_name),
os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, 'RAxML_result.%s' % output_name)):
shutil.copy(os.path.join(tempdir, 'RAxML_result.%s' % output_name),
os.path.abspath(output_dir))
if os.path.exists(os.path.join(tempdir, 'RAxML_log.%s' % output_name)):
shutil.copy(os.path.join(tempdir, 'RAxML_log.%s' % output_name),
os.path.abspath(output_dir))
if not keep_tmp:
sysutils.remove_tempdir(tempdir, 'build_tree', quiet, logfile)
cmd3 = [
'echo',
'Stage completed. Output files are located here: %s\n' %
os.path.abspath(output_dir)
]
sysutils.command_runner([
cmd3,
], 'build_tree', quiet, logfile, debug)