def gatk_variant_selection(self, sample, in1, out, reference, parameters,
exe):
"""Select a subset of variants from a VCF file GATK."""
if exe == 1:
os.chdir(self.home)
args = [
self.gatk, 'SelectVariants', '-R', reference, '-V', in1, '-O',
out, '-select-type', parameters
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[GATK SelectVariants] Sample {sample} Selecting a subset of variants was successful.',
logger_error_mes=
f'[GATK SelectVariants] For some reason Selecting a subset of variants of {sample} '
f'was not successful.',
runtime_error_mes=f'GATK SelectVariants failed ({sample}).')
return out
def picard_md(self, sample, in1, exe):
"""Marking duplicates with Picard."""
out = f'{self.local_output}/GATK/{sample}.Sorted.dedup.bam'
if exe == 1:
os.chdir(self.home)
out_duplicated_metric = f'{self.local_output}/GATK/{sample}.sorted.dedup.metrics.txt'
args = [
self.picard, 'MarkDuplicates', f'INPUT={in1}', f'OUTPUT={out}',
f'METRICS_FILE={out_duplicated_metric}'
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[MD] Sample {sample} Marking duplicates was successfully.',
logger_error_mes=
f'[MD] For some reason the Marking duplicates of {sample} was not successful.',
runtime_error_mes=f'Marking duplicates failed ({sample}).')
return out
def picard_rg(self, sample, in1, ref_organism, exe):
"""Add or replace read groups with Picard."""
out = f'{self.local_output}/GATK/{sample}.RG.bam'
if exe == 1:
os.chdir(self.home)
args = [
self.picard, 'AddOrReplaceReadGroups', f'INPUT={in1}',
f'OUTPUT={out}', f'RGSM={sample}', f'RGPU=none',
f'RGLB={ref_organism}', f'RGPL=ILLUMINA'
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=f'[RG] Sample {sample} RG was successfully.',
logger_error_mes=
f'[RG] For some reason the RG of {sample} was not successful.',
runtime_error_mes=f'RG failed ({sample}).')
return out
def bwa(self, sample, reference, in1, in2, exe, core):
"""Aligning with BWA."""
out_bam = f'{self.local_output}/aligned/{sample}.bam'
if exe == 1:
os.chdir(self.home)
out_bam = f'{self.local_output}/aligned/{sample}.bam'
bwa = Cmd['bwa']
samtools = Cmd['samtools']
args = ["mem", '-t', core, '-M', reference, in1, in2]
# run BWA
result = (bwa[args] | samtools['fixmate', '-m', '-', '-']
| samtools['sort', '-o', out_bam]) & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=f'[BWA] Sample {sample} aligned successfully.',
logger_error_mes=
f'[BWA] For some reason the alignment of {sample} was not successful.',
runtime_error_mes=f'BWA failed ({sample}).')
return out_bam
def bbduk_single(self, sample, r1_name, r1, exe, n_cores, ktrim, qtrim,
trimq, k, mink, hdist, ftm, chastityfilter, minlen,
adapters):
"""Trimming with BBduk single reads."""
trimmed = f'{self.local_output}/trimmed/BBduk_{r1_name}.fastq'
if exe == 1:
os.chdir(self.home)
args = [
f'in={r1}', f'out={trimmed}',
f'ref={self.local_adapters}/{adapters}', f'ktrim={ktrim}',
f'qtrim={qtrim}', f'trimq={trimq}', f'overwrite=true',
f'k={k}', f'mink={mink}', f'hdist={hdist}', f'tpe', f'tbo',
f'ftm={ftm}', f'chastityfilter={chastityfilter}',
f'minlen={minlen}', f'threads={n_cores}'
]
# run BBduk
result = Cmd['bbduk.sh'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=f'[BBduk] Sample {sample} trimmed successfully.',
logger_error_mes=
f'[BBduk] For some reason the trimming of {sample} was not successful.',
runtime_error_mes=f'BBduk failed ({sample}).')
return trimmed
def gatk_merge_vcfs(self, exe, reference, sample):
"""Gather all created VCFs and combine them."""
out = f"{self.local_output}/GATK/VCFs/{sample}-genotype_variants.vcf.gz"
out_merged = f"{self.local_output}/GATK/VCFs/{sample}-merged_variants.g.vcf.gz"
if exe == 1:
os.chdir(self.home)
relevant_path = "output/GATK/VCFs/"
included_extensions = ['-named_variants.g.vcf']
file_names = [
fn for fn in os.listdir(relevant_path) if any(
fn.endswith(ext) for ext in included_extensions)
]
input_variant_files_list = []
variants = []
for gvcf in file_names:
sam = gvcf.split("-")[0]
if sam in self.ori_samples:
input_variant_files_list.append(gvcf)
variants.append('-V')
variants.append("output/GATK/VCFs/" + gvcf)
args = [
self.gatk, 'CombineGVCFs', variants, '-R', reference, '-O',
out_merged
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[GATK CombineGVCFs] combining g.VCFs was successful.',
logger_error_mes=
f'[GATK CombineGVCFs] For some reason the combining of g.VCFs '
f'was not successful.',
runtime_error_mes=f'GATK CombineGVCFs failed.')
args = [
self.gatk, 'GenotypeGVCFs', '-V', out_merged, '-R', reference,
'-O', out
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=f'[GATK GenotypeGVCFs] Genotyping was successful.',
logger_error_mes=
f'[GATK GenotypeGVCFs] For some reason genotyping '
f'was not successful.',
runtime_error_mes=f'GATK GenotypeGVCFs failed.')
return out
def fastqc_single(self, status, sample, in1):
"""Create a report of fasta file quality."""
os.chdir(self.local_input)
result = Cmd['fastqc'][in1, '-o',
f'{self.local_output}/fastqc/{status}/'] & TEE(
retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=f'[Fastqc] Sample {sample} analysed successfully.',
logger_error_mes=
f'[FastQC] For some reason the analysis of {sample} was not successful.',
runtime_error_mes=f'FastQC failed ({sample}).')
def gatk_BaseRecalibrator(self, sample, bam, ref_fasta, exe, ref_vcf):
"""Builds a model and then applies it to get new quality scores"""
bqsr_recal_table = f"{sample}_recal_data.table"
out = f"{sample}.bqsrCal.bam"
if exe == 1:
os.chdir(self.home)
args = [
self.gatk, 'BaseRecalibrator', '-R', ref_fasta, '-I', bam,
'--use-original-qualities', '-known-sites', ref_vcf, '-O',
bqsr_recal_table
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[GATK] Sample {sample} Base model calibration was successful.',
logger_error_mes=
f'[GATK] For some reason base model calibration for {sample} was not successful.',
runtime_error_mes=f'Base model calibration failed ({sample}).')
# Run ApplyBQSR
args = [
self.gatk, 'ApplyBQSR', '--add-output-sam-program-record',
'-R', ref_fasta, '-I', bam, '--use-original-qualities',
'--bqsr-recal-file', bqsr_recal_table, '-O', out
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[GATK] Sample {sample} quality score correction was successful.',
logger_error_mes=
f'[GATK] For some quality score correction of {sample} was not successful.',
runtime_error_mes=f'Quality score correction failed ({sample}).'
)
return out
def gatk_variants_to_table(self, in1, sample, exe):
"""Transforms a .vcf file into a table"""
out = f"{self.local_output}/GATK/tables/{sample}-raw.snps.table"
if exe == 1:
os.chdir(self.home)
args = [
self.gatk,
'VariantsToTable',
'-V',
in1,
'-F',
'CHROM',
'-F',
'POS',
'-F',
'REF',
'-F',
'ALT',
'-GF',
'DP',
'-GF',
'AD',
'-GF',
'GQ',
'-GF',
'PL',
'-O',
out,
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[GATK VariantsToTable] Sample {sample} table creation was successful.',
logger_error_mes=
f'[GATK VariantsToTable] For some reason the table creation of {sample} '
f'was not successful.',
runtime_error_mes=f'GATK VariantsToTable failed ({sample}).')
return out
def samtools_index(self, sample, in1, exe):
"""Create index with samtools."""
if exe == 1:
os.chdir(self.home)
samtools = Cmd['samtools']
result = samtools['index', in1] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[Samtools index] Sample {sample} indexing successfully.',
logger_error_mes=
f'[Samtools index] For some reasonthe indexing of {sample} was not successful.',
runtime_error_mes=f'Samtools index failed ({sample}).')
def make_references(self, fasta_ref_file, ref_vcf):
"""Crete bwa index, index referencing and dictionary of ref. sequence."""
ref_name = fasta_ref_file.split('.')[:-1]
if len(ref_name) > 1:
ref_name = '.'.join(ref_name)
else:
ref_name = ref_name[0]
fasta_ref_file_path = f"../input/references/{fasta_ref_file}"
vcf_ref_file_path = f"../input/references/{ref_vcf}"
ref_name_path = f"../input/references/{ref_name}"
# Create bwa index
result = Cmd["bwa"]["index", fasta_ref_file_path] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=f'[BWA index] index created.',
logger_error_mes=
f'[BWA index] For some reason the creation of index was not successful.',
runtime_error_mes=f'bwa index failed ({fasta_ref_file}).')
# Make index referencing
result = Cmd["samtools"]["faidx",
fasta_ref_file_path] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=f'[Samtools faidx] indexing referencing done.',
logger_error_mes=
f'[Samtools faidx] For some reason indexing referencing was not successful.',
runtime_error_mes=f'Samtools faidx failed ({fasta_ref_file}).')
# Make dictionary
if not os.path.isfile(ref_name_path + '.dict'):
args = [
self.picard, 'CreateSequenceDictionary',
f'REFERENCE={fasta_ref_file_path}'
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[Picard CreateSequenceDictionary] Sequence dictionary created.',
logger_error_mes=
f'[Picard CreateSequenceDictionary] For some reason creation '
f'of dictionary was not successful.',
runtime_error_mes=
f'Picard CreateSequenceDictionary failed ({fasta_ref_file}).')
# VCF index creation
if not os.path.isfile(ref_name_path + '.tbi'):
args = [self.gatk, 'IndexFeatureFile', '-I', vcf_ref_file_path]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[GATK] Index creation with IndexFeatureFile for {ref_vcf} was successful.',
logger_error_mes=
f'[GATK] For some reason index creation with IndexFeatureFile for {ref_vcf} was not successful.',
runtime_error_mes=
f'IndexFeatureFile Index creation failed ({ref_vcf}).')
def gatk_haplotype_caller(self, sample, in1, reference, exe, ploidy,
confidence):
"""Call germline SNPs and indels via local re-assembly of haplotypes with GATK."""
out = f'{self.local_output}/GATK/VCFs/{sample}-raw_variants.g.vcf'
out_named = f'{self.local_output}/GATK/VCFs/{sample}-named_variants.g.vcf'
if exe == 1:
os.chdir(self.home)
args = [
self.gatk, 'HaplotypeCaller', '-R', reference, '-I', in1, '-O',
out, '-ploidy', ploidy, '-ERC', 'GVCF', '-stand-call-conf',
confidence
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[GATK HaplotypeCaller] Sample {sample} Calling germline SNPs and indels was successful.',
logger_error_mes=
f'[GATK HaplotypeCaller] For some reason Calling germline SNPs and indels of {sample} '
f'was not successful.',
runtime_error_mes=f'GATK HaplotypeCaller failed ({sample}).')
args = [
self.picard, 'RenameSampleInVcf', f'INPUT={out}',
f'OUTPUT={out_named}', f'NEW_SAMPLE_NAME={sample}'
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[PICARD RenameSampleInVcf] Sample {sample} .gvcf sample naming was successful.',
logger_error_mes=
f'[PICARD RenameSampleInVcf] For some reason .gvcf sample naming of {sample} '
f'was not successful.',
runtime_error_mes=f'PICARD RenameSampleInVcf failed ({sample}).'
)
args = [
self.gatk, 'ValidateVariants', '--validate-GVCF', '-R',
reference, '-V', out
]
result = Cmd['java']['-jar'][args] & TEE(retcode=None)
interpret_result(
log=self.logger,
so_logger=self.st_logger,
report=result,
logger_mes=
f'[GATK ValidateVariants] Sample {sample} validation was successful.',
logger_error_mes=
f'[GATK ValidateVariants] For some reason validation of {sample} '
f'was not successful.',
runtime_error_mes=f'GATK ValidateVariants failed ({sample}).')
return out_named