def rsem_index(rsem_index_executable, fasta_input, bowtie_info, params):
'''
This module will create the rsem indexes at params.index_destination using
RSEM_INDEX_EXECUTABLE. If FASTA_INPUT = True, it will use the bowtie version
to make bowtie indexes as well.
bowtie_info is a tuple of (bowtie_path, bowtie_version)
params contains
index_destination - Folder to store the indexes
n - number of cores to use
genome_fasta - path to genomic fasta file. Can also specify DOWNLOAD.
genome_version - hg19/hg38
logfile - Open file handle to a log file
RETURN VALUES
index_path - Path to directory where nidexes were stored
'''
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Creating rsem references...',
file=params.logfile)
index_path = os.path.abspath(params.index_destination)
# If the directory doesn't exist, create it
if not os.path.exists(index_path):
prepare.py_mkdir(index_path)
if params.genome_fasta == 'DOWNLOAD':
params.genome_fasta = prepare.get_genome(params.genome_version,
index_path,
params.tbtf_executable,
params.logfile)
else:
params.genome_fasta = pi_errors.test_param_value(
params.genome_fasta, 'Genomic Fasta', '--genome_fasta',
params.logfile)
# If the gtf file is required, download it
gencode_file = prepare.get_gtf(params.genome_version, index_path,
params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Running rsem-prepare-reference on fasta reference.',
file=params.logfile)
rsem_prepref_call = [rsem_index_executable] # base call
rsem_prepref_call.extend(['--gtf', gencode_file]) # gtf file
if fasta_input:
rsem_prepref_call.extend([
''.join(['--', bowtie_version]),
''.join(['--', bowtie_version, '-path']), bowtie_path
])
else:
rsem_prepref_call.append('--no-bowtie')
rsem_prepref_call.append(params.genome_fasta)
rsem_prepref_call.extend(
[''.join([index_path, '/', params.genome_version])])
print(rsem_prepref_call, file=params.logfile)
return_value = call(rsem_prepref_call)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Indexing Failed', params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Indexing completed.',
file=params.logfile)
return index_path
def rsem_index(rsem_index_executable, fasta_input, bowtie_info, params):
'''
This module will create the rsem indexes at params.index_destination using
RSEM_INDEX_EXECUTABLE. If FASTA_INPUT = True, it will use the bowtie version
to make bowtie indexes as well.
bowtie_info is a tuple of (bowtie_path, bowtie_version)
params contains
index_destination - Folder to store the indexes
n - number of cores to use
genome_fasta - path to genomic fasta file. Can also specify DOWNLOAD.
genome_version - hg19/hg38
logfile - Open file handle to a log file
RETURN VALUES
index_path - Path to directory where nidexes were stored
'''
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Creating rsem references...', file=params.logfile)
index_path = os.path.abspath(params.index_destination)
# If the directory doesn't exist, create it
if not os.path.exists(index_path):
prepare.py_mkdir(index_path)
if params.genome_fasta == 'DOWNLOAD':
params.genome_fasta = prepare.get_genome(params.genome_version,
index_path,
params.tbtf_executable,
params.logfile)
else:
params.genome_fasta = pi_errors.test_param_value(params.genome_fasta,
'Genomic Fasta',
'--genome_fasta',
params.logfile)
# If the gtf file is required, download it
gencode_file = prepare.get_gtf(params.genome_version, index_path,
params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Running rsem-prepare-reference on fasta reference.',
file=params.logfile)
rsem_prepref_call = [rsem_index_executable] # base call
rsem_prepref_call.extend(['--gtf', gencode_file]) # gtf file
if fasta_input:
rsem_prepref_call.extend([''.join(['--', bowtie_version]),
''.join(['--', bowtie_version, '-path']),
bowtie_path])
else:
rsem_prepref_call.append('--no-bowtie')
rsem_prepref_call.append(params.genome_fasta)
rsem_prepref_call.extend([''.join([index_path, '/',
params.genome_version])])
print(rsem_prepref_call, file=params.logfile)
return_value = call(rsem_prepref_call)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Indexing Failed', params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Indexing completed.', file=params.logfile)
return index_path
def star_indexing(star_executable, read_length, params):
'''
This module indexes a genome using STAR_EXECUTABLE using READ_LENGTH to set
edge size.
params contains
index_destination - The location where the index should be stored
logfile - Open file handle to a log file
genome_version - hg19/hg38
n - number of cores to use
tbtf_executable - path to twoBitToFa
RETURN VALUES
index_path - path ot the directory where indexes were stored
'''
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Indexing fasta...',
file=params.logfile)
params.index_destination = os.path.abspath(params.index_destination)
if not os.path.exists(params.index_destination):
prepare.py_mkdir(params.index_destination)
edge_size = max(50, int(round(read_length / 50, 0) * 50)) # minimum edge
# size = 50
index_path = ''.join(
[params.index_destination, '/STAR_',
str(edge_size), '_references'])
if not os.path.exists(index_path): # make reference based on edge size
prepare.py_mkdir(index_path)
genome_fasta = prepare.get_genome(params.genome_version, index_path,
params.tbtf_executable, params.logfile)
gencode_file = prepare.get_gtf(params.genome_version, index_path,
params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Running STAR index on fasta reference.',
file=params.logfile)
starindex_call = [star_executable] # Base call
starindex_call.extend(['--runThreadN', str(params.n)]) # Threads
starindex_call.extend(['--runMode', 'genomeGenerate']) # Indexing module
starindex_call.extend(['--genomeDir', index_path]) # index directory
starindex_call.extend(['--genomeFastaFiles', genome_fasta]) # Genomic fa
starindex_call.extend(['--sjdbGTFfile', gencode_file]) # gencode annots
starindex_call.extend(['--sjdbOverhang', str(read_length)]) # edge size
print(starindex_call, file=params.logfile)
return_value = call(starindex_call)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Indexing Failed', params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Indexing completed.',
file=params.logfile)
return index_path
def star_indexing(star_executable, read_length, params):
'''
This module indexes a genome using STAR_EXECUTABLE using READ_LENGTH to set
edge size.
params contains
index_destination - The location where the index should be stored
logfile - Open file handle to a log file
genome_version - hg19/hg38
n - number of cores to use
tbtf_executable - path to twoBitToFa
RETURN VALUES
index_path - path ot the directory where indexes were stored
'''
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Indexing fasta...', file=params.logfile)
params.index_destination = os.path.abspath(params.index_destination)
if not os.path.exists(params.index_destination):
prepare.py_mkdir(params.index_destination)
edge_size = max(50, int(round(read_length / 50, 0) * 50)) # minimum edge
# size = 50
index_path = ''.join([params.index_destination, '/STAR_',
str(edge_size), '_references'])
if not os.path.exists(index_path): # make reference based on edge size
prepare.py_mkdir(index_path)
genome_fasta = prepare.get_genome(params.genome_version, index_path,
params.tbtf_executable,
params.logfile)
gencode_file = prepare.get_gtf(params.genome_version, index_path,
params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Running STAR index on fasta reference.', file=params.logfile)
starindex_call = [star_executable] # Base call
starindex_call.extend(['--runThreadN', str(params.n)]) # Threads
starindex_call.extend(['--runMode', 'genomeGenerate']) # Indexing module
starindex_call.extend(['--genomeDir', index_path]) # index directory
starindex_call.extend(['--genomeFastaFiles', genome_fasta]) # Genomic fa
starindex_call.extend(['--sjdbGTFfile', gencode_file]) # gencode annots
starindex_call.extend(['--sjdbOverhang', str(read_length)]) # edge size
print(starindex_call, file=params.logfile)
return_value = call(starindex_call)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Indexing Failed', params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Indexing completed.', file=params.logfile)
return index_path
def main():
'''
This wrapper script will run the entire alignment pipeline for genomic DNA
(WGS or WXS) from alignment of fastqs, to sorting, indexing, and Read Group
incorporation. The wrapper can even download and produce bwa references if
required. The wrapper requires
1. bwa (For aligning reads)
2. java (For picard)
3. picard tools (For read groups)
4. samtools (For sam/bam manipulation)
5. twoBitToFa from the kent tools library (For extracting the reference
genome in case indexing is required)
Unless specified, the program will look for default executables on $PATH.
The program DOES NOT look for jar files and they are required to be
passed during execution.
'''
# Parse the arguments using prepare.parse_args()
params = prepare.parse_args(main.__doc__, 'bwa', 'bwa_alignment')
# Params ERROR handling
# The memory option for java should be of the form Xmx10G or Xmx10M
if not (params.java_Xmx.endswith('G') or params.java_Xmx.endswith('M')):
raise pi_errors.ParameterError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Please use a suitable value for --Xmx.', params.logfile)
params.bwa_executable = pi_errors.test_param_value(params.bwa_executable,
'bwa',
'--bwa',
params.logfile)
params.samtools_executable = pi_errors.test_param_value(
params.samtools_executable, 'samtools', '--samtools', params.logfile)
params.java_executable = pi_errors.test_param_value(params.java_executable,
'java',
'--java',
params.logfile)
# If Indexing is required, does twoBitToFa point to a valid file?
if params.index_location is None:
params.tbtf_executable = pi_errors.test_param_value(
params.tbtf_executable, 'twoBitToFa', '--twoBitToFa',
params.logfile)
if not params.picard_jar.endswith('jar'):
raise pi_errors.ParameterError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Please specify a valid jar file for picard!', params.logfile)
else:
params.picard_jar = pi_errors.test_param_value(params.picard_jar,
'picard',
'--picard_jar',
params.logfile)
if params.RGID is None:
params.RGID = params.file_prefix
#read_group = ''.join(['\'@RG\\tID:', params.RGID, '\\tPL:ILLUMINA\\tSM:',
# params.sample_type, '\''])
# Check for indexes. If the user has specified that indexes need to
# be created then do so.
if params.index_location is None:
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Indexing fasta...', file=params.logfile)
if not os.path.exists(params.index_destination):
prepare.py_mkdir(params.index_destination)
index_path = params.index_destination
genome_fasta = prepare.get_genome(params.genome_version, index_path,
params.twoBitToFa_executable,
params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Running BWA index on fasta reference.', file=params.logfile)
return_value = call([params.bwa_executable, 'index', genome_fasta])
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': bwa index failed.', params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Running samtools faidx.', file=params.logfile)
return_value = call([params.samtools_executable, 'faidx', genome_fasta])
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': samtools faidx failed', params.logfile)
index_prefix = genome_fasta
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Indexing completed.', file=params.logfile)
else:
if params.index_location.endswith('.fa'):
assert os.path.exists(params.index_location), 'Index file not found'
index_prefix = params.index_location
else:
fastas = [x for x in os.listdir(params.index_location) if
x.endswith(".fa")]
if len(fastas) == 1:
index_prefix = "".join([params.index_location, '/', fastas[0]])
elif len(fastas) == 0:
raise pi_errors.InputFileError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': No valid fasta found in provided index folder',
params.logfile)
else:
raise pi_errors.InputFileError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
':Multiple fastas found in provided index folder. Try ' + \
'running with --index_location /path/to/file/filename.fa',
params.logfile)
# Move to working directory before doing I/O intensive alignment
os.chdir(params.working_dir)
# Align reads to sam file
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': Aligning' +
' reads to reference.', file=params.logfile)
bwa_call = [params.bwa_executable, 'mem'] # base call
bwa_call.extend(['-t', str(params.n)]) # Number of threads
#bwa_call.extend(['-R', read_group]) # Read group
bwa_call.append(index_prefix) # bwa index
bwa_call.append(''.join([params.file_path, '/', params.file_prefix,
'_1.fastq']))
bwa_call.append(''.join([params.file_path, '/', params.file_prefix,
'_2.fastq']))
print(' '.join(bwa_call), file=params.logfile)
with open(''.join([params.file_prefix, '.sam']), 'w') as samfile, \
open(''.join([params.file_prefix, '_bwa_log.txt']), 'w') as logfile:
return_value = call(bwa_call, stdout=samfile, stderr=logfile)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': bwa mem failed.', params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Alignment completed. Converting to bam', file=params.logfile)
# Convert the sam to a bam file
with open(''.join([params.file_prefix, '.bam']), 'w') as bamfile:
call([params.samtools_executable, 'view', '-bS',
''.join([params.file_prefix, '.sam'])], stdout=bamfile)
call(['rm', ''.join([params.file_prefix, '.sam'])])
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': bam file' +
' created. Preparing file for inserting RG into header.',
file=params.logfile)
# Fix PG line
sam_header = check_output([params.samtools_executable, 'view', '-H',
''.join([params.file_prefix, '.bam'])])
sam_header = sam_header.strip().split('\n') # Strip whitespace and separate
pg_line = sam_header[-1].split('\t') # Grab @PG line + split by tab
# Then remove the CL field form the PG line
sam_header[-1] = '\t'.join([x for x in pg_line if not x.startswith('CL')])
with open(''.join([params.file_prefix, '_sam.header']), 'w') as hdr_file:
print('\n'.join(sam_header), file=hdr_file)
with open(''.join([params.file_prefix, '_fixPG.bam']), 'w') as \
fixpg_bamfile:
return_value = call([params.samtools_executable, 'reheader',
''.join([params.file_prefix, '_sam.header']),
''.join([params.file_prefix, '.bam'])],
stdout=fixpg_bamfile)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': samtools reheader failed', params.logfile)
call(['rm', ''.join([params.file_prefix, '.bam']),
''.join([params.file_prefix, '_sam.header'])])
# Sort and Index the _fixPG.bam file
return_value = call([params.samtools_executable, 'sort',
''.join([params.file_prefix, '_fixPG.bam']),
''.join([params.file_prefix, '_fixPG_sorted'])])
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': samtools sort failed.', params.logfile)
return_value = call([params.samtools_executable, 'index',
''.join([params.file_prefix, '_fixPG_sorted.bam'])])
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': samtools index failed.', params.logfile)
call(['rm', ''.join([params.file_prefix, '_fixPG.bam'])])
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Inserting @RG tag into header.', file=params.logfile)
# Reheader the indexed _fixPG_sorted.bam to prepare for mutect
picard_call = [params.java_executable, ''.join(['-Xmx', params.java_Xmx]),
'-jar'] # Base java call
picard_call.append(params.picard_jar) # picard
picard_call.append('AddOrReplaceReadGroups') # module
picard_call.append('CREATE_INDEX=true')
picard_call.append(''.join(['I=', params.file_prefix, '_fixPG_sorted.bam']))
picard_call.append(''.join(['O=', params.file_prefix,
'_fixPG_sorted_reheader.bam']))
picard_call.append('SO=coordinate')
picard_call.append('ID=1')
picard_call.append(''.join(['LB=', params.file_prefix]))
picard_call.append('PL=ILLUMINA')
picard_call.append('PU=12345')
picard_call.append(''.join(['SM=', params.sample_type]))
with open(''.join([params.file_prefix, '_picard_log.txt']), 'w') as logfile:
return_value = call(picard_call, stdout=logfile)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': picard AddOrReplaceReadGroups failed.', params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': @RG ' +
'inserted. Indexing bam', file=params.logfile)
# Index _fixPG_sorted_reheader.bam file
return_value = call([params.samtools_executable, 'index',
''.join([params.file_prefix,
'_fixPG_sorted_reheader.bam'])])
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': samtools index failed.', params.logfile)
# Remove intermediate files
call(['rm', ''.join([params.file_prefix, '_fixPG_sorted.bam']),
''.join([params.file_prefix, '_fixPG_sorted.bam.bai'])])
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Alignment completed. Finishing up...', params.logfile)
# Move files from temp directory to outdir
prepare.move_output(params)
print('RESULT ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': Process ' +
'completed', file=params.logfile)
params.logfile.close()
def main():
'''
This wrapper script will run the entire alignment pipeline for genomic DNA
(WGS or WXS) from alignment of fastqs, to sorting, indexing, and Read Group
incorporation. The wrapper can even download and produce bwa references if
required. The wrapper requires
1. bwa (For aligning reads)
2. java (For picard)
3. picard tools (For read groups)
4. samtools (For sam/bam manipulation)
5. twoBitToFa from the kent tools library (For extracting the reference
genome in case indexing is required)
Unless specified, the program will look for default executables on $PATH.
The program DOES NOT look for jar files and they are required to be
passed during execution.
'''
# Parse the arguments using prepare.parse_args()
params = prepare.parse_args(main.__doc__, 'bwa', 'bwa_alignment')
# Params ERROR handling
# The memory option for java should be of the form Xmx10G or Xmx10M
if not (params.java_Xmx.endswith('G') or params.java_Xmx.endswith('M')):
raise pi_errors.ParameterError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Please use a suitable value for --Xmx.', params.logfile)
params.bwa_executable = pi_errors.test_param_value(params.bwa_executable,
'bwa', '--bwa',
params.logfile)
params.samtools_executable = pi_errors.test_param_value(
params.samtools_executable, 'samtools', '--samtools', params.logfile)
params.java_executable = pi_errors.test_param_value(
params.java_executable, 'java', '--java', params.logfile)
# If Indexing is required, does twoBitToFa point to a valid file?
if params.index_location is None:
params.tbtf_executable = pi_errors.test_param_value(
params.tbtf_executable, 'twoBitToFa', '--twoBitToFa',
params.logfile)
if not params.picard_jar.endswith('jar'):
raise pi_errors.ParameterError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Please specify a valid jar file for picard!', params.logfile)
else:
params.picard_jar = pi_errors.test_param_value(params.picard_jar,
'picard',
'--picard_jar',
params.logfile)
if params.RGID is None:
params.RGID = params.file_prefix
#read_group = ''.join(['\'@RG\\tID:', params.RGID, '\\tPL:ILLUMINA\\tSM:',
# params.sample_type, '\''])
# Check for indexes. If the user has specified that indexes need to
# be created then do so.
if params.index_location is None:
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Indexing fasta...',
file=params.logfile)
if not os.path.exists(params.index_destination):
prepare.py_mkdir(params.index_destination)
index_path = params.index_destination
genome_fasta = prepare.get_genome(params.genome_version, index_path,
params.twoBitToFa_executable,
params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Running BWA index on fasta reference.',
file=params.logfile)
return_value = call([params.bwa_executable, 'index', genome_fasta])
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': bwa index failed.', params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Running samtools faidx.',
file=params.logfile)
return_value = call(
[params.samtools_executable, 'faidx', genome_fasta])
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': samtools faidx failed', params.logfile)
index_prefix = genome_fasta
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Indexing completed.',
file=params.logfile)
else:
if params.index_location.endswith('.fa'):
assert os.path.exists(
params.index_location), 'Index file not found'
index_prefix = params.index_location
else:
fastas = [
x for x in os.listdir(params.index_location)
if x.endswith(".fa")
]
if len(fastas) == 1:
index_prefix = "".join([params.index_location, '/', fastas[0]])
elif len(fastas) == 0:
raise pi_errors.InputFileError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': No valid fasta found in provided index folder',
params.logfile)
else:
raise pi_errors.InputFileError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
':Multiple fastas found in provided index folder. Try ' + \
'running with --index_location /path/to/file/filename.fa',
params.logfile)
# Move to working directory before doing I/O intensive alignment
os.chdir(params.working_dir)
# Align reads to sam file
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') +
': Aligning' + ' reads to reference.',
file=params.logfile)
bwa_call = [params.bwa_executable, 'mem'] # base call
bwa_call.extend(['-t', str(params.n)]) # Number of threads
#bwa_call.extend(['-R', read_group]) # Read group
bwa_call.append(index_prefix) # bwa index
bwa_call.append(''.join(
[params.file_path, '/', params.file_prefix, '_1.fastq']))
bwa_call.append(''.join(
[params.file_path, '/', params.file_prefix, '_2.fastq']))
print(' '.join(bwa_call), file=params.logfile)
with open(''.join([params.file_prefix, '.sam']), 'w') as samfile, \
open(''.join([params.file_prefix, '_bwa_log.txt']), 'w') as logfile:
return_value = call(bwa_call, stdout=samfile, stderr=logfile)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': bwa mem failed.', params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Alignment completed. Converting to bam',
file=params.logfile)
# Convert the sam to a bam file
with open(''.join([params.file_prefix, '.bam']), 'w') as bamfile:
call([
params.samtools_executable, 'view', '-bS', ''.join(
[params.file_prefix, '.sam'])
],
stdout=bamfile)
call(['rm', ''.join([params.file_prefix, '.sam'])])
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') +
': bam file' +
' created. Preparing file for inserting RG into header.',
file=params.logfile)
# Fix PG line
sam_header = check_output([
params.samtools_executable, 'view', '-H',
''.join([params.file_prefix, '.bam'])
])
sam_header = sam_header.strip().split(
'\n') # Strip whitespace and separate
pg_line = sam_header[-1].split('\t') # Grab @PG line + split by tab
# Then remove the CL field form the PG line
sam_header[-1] = '\t'.join([x for x in pg_line if not x.startswith('CL')])
with open(''.join([params.file_prefix, '_sam.header']), 'w') as hdr_file:
print('\n'.join(sam_header), file=hdr_file)
with open(''.join([params.file_prefix, '_fixPG.bam']), 'w') as \
fixpg_bamfile:
return_value = call([
params.samtools_executable, 'reheader', ''.join([
params.file_prefix, '_sam.header'
]), ''.join([params.file_prefix, '.bam'])
],
stdout=fixpg_bamfile)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': samtools reheader failed', params.logfile)
call([
'rm', ''.join([params.file_prefix, '.bam']),
''.join([params.file_prefix, '_sam.header'])
])
# Sort and Index the _fixPG.bam file
return_value = call([
params.samtools_executable, 'sort',
''.join([params.file_prefix,
'_fixPG.bam']), ''.join([params.file_prefix, '_fixPG_sorted'])
])
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': samtools sort failed.', params.logfile)
return_value = call([
params.samtools_executable, 'index',
''.join([params.file_prefix, '_fixPG_sorted.bam'])
])
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': samtools index failed.', params.logfile)
call(['rm', ''.join([params.file_prefix, '_fixPG.bam'])])
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Inserting @RG tag into header.',
file=params.logfile)
# Reheader the indexed _fixPG_sorted.bam to prepare for mutect
picard_call = [
params.java_executable, ''.join(['-Xmx', params.java_Xmx]), '-jar'
] # Base java call
picard_call.append(params.picard_jar) # picard
picard_call.append('AddOrReplaceReadGroups') # module
picard_call.append('CREATE_INDEX=true')
picard_call.append(''.join(['I=', params.file_prefix,
'_fixPG_sorted.bam']))
picard_call.append(''.join(
['O=', params.file_prefix, '_fixPG_sorted_reheader.bam']))
picard_call.append('SO=coordinate')
picard_call.append('ID=1')
picard_call.append(''.join(['LB=', params.file_prefix]))
picard_call.append('PL=ILLUMINA')
picard_call.append('PU=12345')
picard_call.append(''.join(['SM=', params.sample_type]))
with open(''.join([params.file_prefix, '_picard_log.txt']),
'w') as logfile:
return_value = call(picard_call, stdout=logfile)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': picard AddOrReplaceReadGroups failed.', params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': @RG ' +
'inserted. Indexing bam',
file=params.logfile)
# Index _fixPG_sorted_reheader.bam file
return_value = call([
params.samtools_executable, 'index',
''.join([params.file_prefix, '_fixPG_sorted_reheader.bam'])
])
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': samtools index failed.', params.logfile)
# Remove intermediate files
call([
'rm', ''.join([params.file_prefix, '_fixPG_sorted.bam']),
''.join([params.file_prefix, '_fixPG_sorted.bam.bai'])
])
print(
'PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Alignment completed. Finishing up...', params.logfile)
# Move files from temp directory to outdir
prepare.move_output(params)
print('RESULT ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': Process ' +
'completed',
file=params.logfile)
params.logfile.close()
def main():
'''
This wrapper script will run the tool PHLAT within the phlat docker
container for the precision immuno project. The wrapper requires:
1. PHLAT.py
2. bowtie2
3. gdown.pl (For donwloading the PHLAT Index - available from
https://raw.githubusercontent.com/Nanolx/patchimage/master/tools/gdown.pl)
Unless specified, the program will look for default executables on $PATH.
The program DOES NOT look for jar files and they are required to be
passed during execution.
'''
# Parse the arguments using prepare.parse_args()
params = prepare.parse_args(main.__doc__, 'phlat', 'MHC_typing')
# params ERROR handling
if not params.phlat_executable.endswith('PHLAT.py'):
params.phlat_executable = '/'.join([params.phlat_executable,
'PHLAT.py'])
params.phlat_executable = pi_errors.test_param_value(
params.phlat_executable, 'PHLAT', '--phlat', params.logfile)
params.bowtie2_executable = pi_errors.test_param_value(
params.bowtie2_executable, 'bowtie2', '--bowtie2', params.logfile)
phlat_dir = os.path.split(os.path.split(params.phlat_executable)[0])[0]
params.gdownpl_executable = pi_errors.test_param_value(
params.gdownpl_executable, 'gdown.pl', '--gdownpl', params.logfile)
if params.index_location is None:
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') +': ' +
'Downloading Indexes...', file=params.logfile)
params.index_destination = os.path.abspath(params.index_destination)
if not os.path.exists(params.index_destination):
prepare.py_mkdir(params.index_destination)
getindex_call = [params.gdownpl_executable, 'https://drive.google.com' +
'/uc?export=download&confirm=yAjx&id=0Bz-w5tutuZIYY3' +
'h5YlMzTjhnbGM', ''.join([params.index_destination,
'/index4phlat.tar.gz'])]
print(getindex_call, file=params.logfile)
return_value = call(getindex_call)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Could not download indexes. Try manually downloading.',
params.logfile)
extract_call = ['tar', '-C', params.index_destination, '-zxvf',
'/'.join([params.index_destination,
'index4phlat.tar.gz'])]
return_value = call(extract_call)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Index4phlat could not be extracted.', params.logfile)
else:
call(['rm', '/'.join([params.index_destination,
'index4phlat.tar.gz'])])
index_path = '/'.join([params.index_destination, 'index4phlat'])
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Indexes Downloaded.', file=params.logfile)
else:
params.index_location = os.path.abspath(params.index_location)
if not os.path.exists(''.join([params.index_location,
'/ucsc.artHLA.1.bt2'])):
raise pi_errors.InputFileError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Index file not found.', params.logfile)
else:
index_path = params.index_location
# Move to working directory before doing I/O intensive alignment
os.chdir(params.working_dir)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') +': ' +
'Begining MHC Haplotyping', file=params.logfile)
system_call = ['/usr/bin/env', 'python2.7', '-O', params.phlat_executable]
system_call.extend(['-1', ''.join([params.file_path, '/',
params.file_prefix, '_1.fastq'])]) # Fq1
system_call.extend(['-2', ''.join([params.file_path, "/",
params.file_prefix, '_2.fastq'])]) # Fq2
system_call.extend(['-index', index_path]) # Index files
system_call.extend(['-b2url', params.bowtie2_executable]) # Bowtie2
system_call.extend(['-tag', ''.join([params.out_prefix])]) # DNA/RNA
system_call.extend(['-e', phlat_dir]) # Phlat directory home
system_call.extend(['-o', params.outdir]) # Output directory
system_call.extend(['-p', str(params.n)]) # Number of threads
# Call the program
return_value = call(system_call)
if return_value != 0:
raise pi_errors.MyRuntimeError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': MHC Haplotyping failed.', params.logfile)
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
'Alignment completed. Finishing up...', file=params.logfile)
# Move files from temp directory to outdir
prepare.move_output(params)
print('RESULT ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': Process ' +
'completed', file=params.logfile)
params.logfile.close()
def process_parameters(params):
'''
This module conducts the error handling for all parmeters passed to the
program.
'''
print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') +
': Processing input parameters.', file=params.logfile)
# Does the input vcf file exist?
if not os.path.exists(''.join([params.file_path, '/', params.file_prefix,
'.vcf'])):
raise pi_errors.InputFileError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Please provide a valid input file using --file_prefix',
params.logfile)
# The memory option for java should be of the form Xmx10G or Xmx10M
if not (params.java_Xmx.endswith('G') or params.java_Xmx.endswith('M')):
raise pi_errors.ParameterError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': Please use a suitable value for --Xmx.', params.logfile)
params.java_executable = pi_errors.test_param_value(params.java_executable,
'java',
'--java',
params.logfile)
# Does the provided snpeff binary provided exist?
params.snpeff_jar = pi_errors.test_param_value(params.snpeff_jar,
'snpeff',
'--snpeff_jar',
params.logfile)
params.use_snpeff_db = False
# Does the user want a snpEff packaged database?
if params.config_file == 'PACKAGED':
params.use_snpeff_db = True
# Has the snpeff reference to be used been provided?
if params.reference_name == None:
raise pi_errors.ParameterError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': --snp_reference is required if --config=PACKAGED.',
params.logfile)
# If a custom databse is desired, does it need to be created?
if params.index_location is None:
# If the user has provided the location to the parent directory of data
# directory, make DATA_DIRECTORY point to data. If they have provided
# the link to data, make INDEX_DESTINATION point to the parent and
# DATA_DIRECTORY point to data.
if os.path.split(params.index_destination.rstrip('/'))[1] != 'data':
params.data_directory = '/'.join([params.index_destination, 'data'])
else:
params.data_directory = params.index_destination
params.index_destination = \
params.index_destination.rstrip('/').rstrip('/data')
# Create the data directory if needed
if not os.path.exists(params.data_directory):
prepare.py_mkdir(params.data_directory)
# If we're using a custom databse, thre is nothing more to do
if params.use_snpeff_db:
return None
# Initialise the reference name
params.reference_name = ''.join([params.genome_version, '_custom'])
# make a variable to gold GENOME_VERSION_custom
genome_folder = '/'.join([params.data_directory, params.reference_name])
prepare.py_mkdir(genome_folder)
# If the genome fasta isn't provided or is provided a wrong value,
# download it
if params.genome_fasta == 'DOWNLOAD' or not \
os.path.exists(params.genome_fasta):
# Does the provided tbtf binary point to a valid file?
params.tbtf_executable = pi_errors.test_param_value(
params.tbtf_executable, 'twoBitToFa', '--twoBitToFa',
params.logfile)
params.genome_fasta = prepare.get_genome(
params.genome_version, genome_folder,
params.tbtf_executable, params.logfile)
# Rename genome fasta
call(['mv', params.genome_fasta, '/'.join([genome_folder,
'sequences.fa'])])
else:
params.genome_fasta = os.path.abspath(params.genome_fasta)
# Link sequencesfa to genome fasta
call(['ln', '-s', '-T', params.genome_fasta,
'/'.join([genome_folder, 'sequences.fa'])])
# Download the gencode GTF file
params.gtf_file = prepare.get_gtf(params.genome_version,
genome_folder,
params.logfile)
# Rename gtf file
call(['mv', params.gtf_file, '/'.join([genome_folder, 'genes.gtf'])])
# If it has been provided, set up the config file
else:
# If the user has provided the location to the parent directory of data
# directory, make DATA_DIRECTORY point to data. If they have provided
# the link to data, make INDEX_LOCATION point to the parent and
# DATA_DIRECTORY point to data.
if os.path.split(params.index_location.rstrip('/'))[1] != 'data':
params.data_directory = '/'.join([params.index_location, 'data'])
else:
params.data_directory = params.index_location
params.index_location = \
params.index_location.rstrip('/').rstrip('/data')
# If we're using a custom databse, thre is nothing more to do
if params.use_snpeff_db:
return None
# If the config file hasn't been provided, is it in INDEX_LOCATION
# AND does GENOME_VERSION_custom exist (i.e. was it created by
# this script?)
if params.config_file is None:
params.config_file = pi_errors.test_param_value(
'/'.join([params.index_location, 'snpEff.config']),
'snpEff.config', '--config', params.logfile)
# Dummy variable to ensure the GENOME_VERSION_custom exists
_ = pi_errors.test_param_value(
''.join([params.data_directory, '/', params.genome_version,
'_custom']), '_'.join([params.genome_version, 'custom'
]), '--snpeff_reference and' +
'--config', params.logfile)
params.reference_name = '_'.join([params.genome_version, 'custom'])
# If a config file has been provided, does it point to a legit file and
# has the reference name also been provided?
else:
params.config_file = pi_errors.test_param_value(
params.config_file, 'snpEff config file', '--config',
params.logfile)
if params.reference_name is None:
raise pi_errors.ParameterError(
dt.now().strftime('%I:%M %p %b %d, %Y') + \
': --snpeff_reference is required if --config points to' + \
' a custom file.', params.logfile)
return None