Example #1
0
def rsem_index(rsem_index_executable, fasta_input, bowtie_info, params):
    '''
    This module will create the rsem indexes at params.index_destination using
    RSEM_INDEX_EXECUTABLE. If FASTA_INPUT = True, it will use the bowtie version
    to make bowtie indexes as well.
    bowtie_info is a tuple of (bowtie_path, bowtie_version)
    params contains
    index_destination - Folder to store the indexes
    n - number of cores to use
    genome_fasta - path to genomic fasta file. Can also specify DOWNLOAD.
    genome_version - hg19/hg38
    logfile - Open file handle to a log file
    RETURN VALUES
    index_path - Path to directory where nidexes were stored
    '''
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Creating rsem references...',
          file=params.logfile)
    index_path = os.path.abspath(params.index_destination)
    #  If the directory doesn't exist, create it
    if not os.path.exists(index_path):
        prepare.py_mkdir(index_path)
    if params.genome_fasta == 'DOWNLOAD':
        params.genome_fasta = prepare.get_genome(params.genome_version,
                                                 index_path,
                                                 params.tbtf_executable,
                                                 params.logfile)
    else:
        params.genome_fasta = pi_errors.test_param_value(
            params.genome_fasta, 'Genomic Fasta', '--genome_fasta',
            params.logfile)
    #  If the gtf file is required, download it
    gencode_file = prepare.get_gtf(params.genome_version, index_path,
                                   params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ':    ' +
          'Running rsem-prepare-reference on fasta reference.',
          file=params.logfile)
    rsem_prepref_call = [rsem_index_executable]  # base call
    rsem_prepref_call.extend(['--gtf', gencode_file])  # gtf file
    if fasta_input:
        rsem_prepref_call.extend([
            ''.join(['--', bowtie_version]),
            ''.join(['--', bowtie_version, '-path']), bowtie_path
        ])
    else:
        rsem_prepref_call.append('--no-bowtie')
    rsem_prepref_call.append(params.genome_fasta)
    rsem_prepref_call.extend(
        [''.join([index_path, '/', params.genome_version])])
    print(rsem_prepref_call, file=params.logfile)
    return_value = call(rsem_prepref_call)
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': Indexing Failed', params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Indexing completed.',
          file=params.logfile)
    return index_path
Example #2
0
def rsem_index(rsem_index_executable, fasta_input, bowtie_info, params):
    '''
    This module will create the rsem indexes at params.index_destination using
    RSEM_INDEX_EXECUTABLE. If FASTA_INPUT = True, it will use the bowtie version
    to make bowtie indexes as well.
    bowtie_info is a tuple of (bowtie_path, bowtie_version)
    params contains
    index_destination - Folder to store the indexes
    n - number of cores to use
    genome_fasta - path to genomic fasta file. Can also specify DOWNLOAD.
    genome_version - hg19/hg38
    logfile - Open file handle to a log file
    RETURN VALUES
    index_path - Path to directory where nidexes were stored
    '''
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Creating rsem references...', file=params.logfile)
    index_path = os.path.abspath(params.index_destination)
    #  If the directory doesn't exist, create it
    if not os.path.exists(index_path):
        prepare.py_mkdir(index_path)
    if params.genome_fasta == 'DOWNLOAD':
        params.genome_fasta = prepare.get_genome(params.genome_version,
                                                 index_path,
                                                 params.tbtf_executable,
                                                 params.logfile)
    else:
        params.genome_fasta = pi_errors.test_param_value(params.genome_fasta,
                                                         'Genomic Fasta',
                                                         '--genome_fasta',
                                                         params.logfile)
    #  If the gtf file is required, download it
    gencode_file = prepare.get_gtf(params.genome_version, index_path,
                                   params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ':    ' +
          'Running rsem-prepare-reference on fasta reference.',
          file=params.logfile)
    rsem_prepref_call = [rsem_index_executable] # base call
    rsem_prepref_call.extend(['--gtf', gencode_file]) # gtf file
    if fasta_input:
        rsem_prepref_call.extend([''.join(['--', bowtie_version]),
                                  ''.join(['--', bowtie_version, '-path']),
                                  bowtie_path])
    else:
        rsem_prepref_call.append('--no-bowtie')
    rsem_prepref_call.append(params.genome_fasta)
    rsem_prepref_call.extend([''.join([index_path, '/',
                                       params.genome_version])])
    print(rsem_prepref_call, file=params.logfile)
    return_value = call(rsem_prepref_call)
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': Indexing Failed', params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Indexing completed.', file=params.logfile)
    return index_path
Example #3
0
def star_indexing(star_executable, read_length, params):
    '''
    This module indexes a genome using STAR_EXECUTABLE using READ_LENGTH to set
    edge size.
    params contains
    index_destination - The location where the index should be stored
    logfile - Open file handle to a log file
    genome_version - hg19/hg38
    n - number of cores to use
    tbtf_executable - path to twoBitToFa
    RETURN VALUES
    index_path - path ot the directory where indexes were stored
    '''
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Indexing fasta...',
          file=params.logfile)
    params.index_destination = os.path.abspath(params.index_destination)
    if not os.path.exists(params.index_destination):
        prepare.py_mkdir(params.index_destination)
    edge_size = max(50, int(round(read_length / 50, 0) * 50))  # minimum edge
    # size = 50
    index_path = ''.join(
        [params.index_destination, '/STAR_',
         str(edge_size), '_references'])
    if not os.path.exists(index_path):  # make reference based on edge size
        prepare.py_mkdir(index_path)
    genome_fasta = prepare.get_genome(params.genome_version, index_path,
                                      params.tbtf_executable, params.logfile)
    gencode_file = prepare.get_gtf(params.genome_version, index_path,
                                   params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ':    ' +
          'Running STAR index on fasta reference.',
          file=params.logfile)
    starindex_call = [star_executable]  # Base call
    starindex_call.extend(['--runThreadN', str(params.n)])  # Threads
    starindex_call.extend(['--runMode', 'genomeGenerate'])  # Indexing module
    starindex_call.extend(['--genomeDir', index_path])  # index directory
    starindex_call.extend(['--genomeFastaFiles', genome_fasta])  # Genomic fa
    starindex_call.extend(['--sjdbGTFfile', gencode_file])  # gencode annots
    starindex_call.extend(['--sjdbOverhang', str(read_length)])  # edge size
    print(starindex_call, file=params.logfile)
    return_value = call(starindex_call)
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': Indexing Failed', params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Indexing completed.',
          file=params.logfile)
    return index_path
Example #4
0
def star_indexing(star_executable, read_length, params):
    '''
    This module indexes a genome using STAR_EXECUTABLE using READ_LENGTH to set
    edge size.
    params contains
    index_destination - The location where the index should be stored
    logfile - Open file handle to a log file
    genome_version - hg19/hg38
    n - number of cores to use
    tbtf_executable - path to twoBitToFa
    RETURN VALUES
    index_path - path ot the directory where indexes were stored
    '''
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Indexing fasta...', file=params.logfile)
    params.index_destination = os.path.abspath(params.index_destination)
    if not os.path.exists(params.index_destination):
        prepare.py_mkdir(params.index_destination)
    edge_size = max(50, int(round(read_length / 50, 0) * 50)) # minimum edge
                                                              # size = 50
    index_path = ''.join([params.index_destination, '/STAR_',
                          str(edge_size), '_references'])
    if not os.path.exists(index_path): # make reference based on edge size
        prepare.py_mkdir(index_path)
    genome_fasta = prepare.get_genome(params.genome_version, index_path,
                                      params.tbtf_executable,
                                      params.logfile)
    gencode_file = prepare.get_gtf(params.genome_version, index_path,
                                   params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ':    ' +
          'Running STAR index on fasta reference.', file=params.logfile)
    starindex_call = [star_executable] # Base call
    starindex_call.extend(['--runThreadN', str(params.n)]) # Threads
    starindex_call.extend(['--runMode', 'genomeGenerate']) # Indexing module
    starindex_call.extend(['--genomeDir', index_path]) # index directory
    starindex_call.extend(['--genomeFastaFiles', genome_fasta]) # Genomic fa
    starindex_call.extend(['--sjdbGTFfile', gencode_file]) # gencode annots
    starindex_call.extend(['--sjdbOverhang', str(read_length)]) # edge size
    print(starindex_call, file=params.logfile)
    return_value = call(starindex_call)
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': Indexing Failed', params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Indexing completed.', file=params.logfile)
    return index_path
Example #5
0
def main():
    '''
    This wrapper script will run the entire alignment pipeline for genomic DNA
    (WGS or WXS) from alignment of fastqs, to sorting, indexing, and Read Group
    incorporation. The wrapper can even download and produce bwa references if
    required. The wrapper requires
    1. bwa (For aligning reads)
    2. java (For picard)
    3. picard tools (For read groups)
    4. samtools (For sam/bam manipulation)
    5. twoBitToFa from the kent tools library (For extracting the reference
            genome in case indexing is required)

    Unless specified, the program will look for default executables on $PATH.
    The program DOES NOT look for jar files and they are required to be
    passed during execution.
    '''
    #  Parse the arguments using prepare.parse_args()
    params = prepare.parse_args(main.__doc__, 'bwa', 'bwa_alignment')

    #  Params ERROR handling
    #  The memory option for java should be of the form Xmx10G or Xmx10M
    if not (params.java_Xmx.endswith('G') or params.java_Xmx.endswith('M')):
        raise pi_errors.ParameterError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': Please use a suitable value for --Xmx.', params.logfile)
    params.bwa_executable = pi_errors.test_param_value(params.bwa_executable,
                                                       'bwa',
                                                       '--bwa',
                                                       params.logfile)
    params.samtools_executable = pi_errors.test_param_value(
        params.samtools_executable, 'samtools', '--samtools', params.logfile)
    params.java_executable = pi_errors.test_param_value(params.java_executable,
                                                        'java',
                                                        '--java',
                                                        params.logfile)
    #  If Indexing is required, does twoBitToFa point to a valid file?
    if params.index_location is None:
        params.tbtf_executable = pi_errors.test_param_value(
            params.tbtf_executable, 'twoBitToFa', '--twoBitToFa',
            params.logfile)
    if not params.picard_jar.endswith('jar'):
        raise pi_errors.ParameterError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': Please specify a valid jar file for picard!', params.logfile)
    else:
        params.picard_jar = pi_errors.test_param_value(params.picard_jar,
                                                       'picard',
                                                       '--picard_jar',
                                                       params.logfile)

    if params.RGID is None:
        params.RGID = params.file_prefix

    #read_group = ''.join(['\'@RG\\tID:', params.RGID, '\\tPL:ILLUMINA\\tSM:',
    #                      params.sample_type, '\''])
    # Check for indexes. If the user has specified that indexes need to
    # be created then do so.
    if params.index_location is None:
        print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
              'Indexing fasta...', file=params.logfile)
        if not os.path.exists(params.index_destination):
            prepare.py_mkdir(params.index_destination)
        index_path = params.index_destination
        genome_fasta = prepare.get_genome(params.genome_version, index_path,
                                          params.twoBitToFa_executable,
                                          params.logfile)
        print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ':    ' +
              'Running BWA index on fasta reference.', file=params.logfile)
        return_value = call([params.bwa_executable, 'index', genome_fasta])
        if return_value != 0:
            raise pi_errors.MyRuntimeError(
                dt.now().strftime('%I:%M %p %b %d, %Y') + \
                ': bwa index failed.', params.logfile)
        print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ':    ' +
              'Running samtools faidx.', file=params.logfile)
        return_value = call([params.samtools_executable, 'faidx', genome_fasta])
        if return_value != 0:
            raise pi_errors.MyRuntimeError(
                dt.now().strftime('%I:%M %p %b %d, %Y') + \
                ': samtools faidx failed', params.logfile)
        index_prefix = genome_fasta
        print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
              'Indexing completed.', file=params.logfile)
    else:
        if params.index_location.endswith('.fa'):
            assert os.path.exists(params.index_location), 'Index file not found'
            index_prefix = params.index_location
        else:
            fastas = [x for x in os.listdir(params.index_location) if
                      x.endswith(".fa")]
            if len(fastas) == 1:
                index_prefix = "".join([params.index_location, '/', fastas[0]])
            elif len(fastas) == 0:
                raise pi_errors.InputFileError(
                    dt.now().strftime('%I:%M %p %b %d, %Y') + \
                    ': No valid fasta found in provided index folder',
                    params.logfile)
            else:
                raise pi_errors.InputFileError(
                    dt.now().strftime('%I:%M %p %b %d, %Y') + \
                    ':Multiple fastas found in provided index folder. Try ' + \
                    'running with --index_location /path/to/file/filename.fa',
                    params.logfile)

    # Move to working directory before doing I/O intensive alignment
    os.chdir(params.working_dir)

    # Align reads to sam file
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': Aligning' +
          ' reads to reference.', file=params.logfile)
    bwa_call = [params.bwa_executable, 'mem'] # base call
    bwa_call.extend(['-t', str(params.n)])  # Number of threads
    #bwa_call.extend(['-R', read_group])  # Read group
    bwa_call.append(index_prefix)  # bwa index
    bwa_call.append(''.join([params.file_path, '/', params.file_prefix,
                             '_1.fastq']))
    bwa_call.append(''.join([params.file_path, '/', params.file_prefix,
                             '_2.fastq']))
    print(' '.join(bwa_call), file=params.logfile)
    with open(''.join([params.file_prefix, '.sam']), 'w') as samfile, \
            open(''.join([params.file_prefix, '_bwa_log.txt']), 'w') as logfile:
        return_value = call(bwa_call, stdout=samfile, stderr=logfile)
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': bwa mem failed.', params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Alignment completed. Converting to bam', file=params.logfile)
    # Convert the sam to a bam file
    with open(''.join([params.file_prefix, '.bam']), 'w') as bamfile:
        call([params.samtools_executable, 'view', '-bS',
              ''.join([params.file_prefix, '.sam'])], stdout=bamfile)
    call(['rm', ''.join([params.file_prefix, '.sam'])])
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': bam file' +
          ' created. Preparing file for inserting RG into header.',
          file=params.logfile)
    # Fix PG line
    sam_header = check_output([params.samtools_executable, 'view', '-H',
                               ''.join([params.file_prefix, '.bam'])])
    sam_header = sam_header.strip().split('\n')  # Strip whitespace and separate
    pg_line = sam_header[-1].split('\t')  # Grab @PG line + split by tab
    # Then remove the CL field form the PG line
    sam_header[-1] = '\t'.join([x for x in pg_line if not x.startswith('CL')])
    with open(''.join([params.file_prefix, '_sam.header']), 'w') as hdr_file:
        print('\n'.join(sam_header), file=hdr_file)
    with open(''.join([params.file_prefix, '_fixPG.bam']), 'w') as \
              fixpg_bamfile:
        return_value = call([params.samtools_executable, 'reheader',
                             ''.join([params.file_prefix, '_sam.header']),
                             ''.join([params.file_prefix, '.bam'])],
                            stdout=fixpg_bamfile)
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': samtools reheader failed', params.logfile)
    call(['rm', ''.join([params.file_prefix, '.bam']),
          ''.join([params.file_prefix, '_sam.header'])])
    # Sort and Index the _fixPG.bam file
    return_value = call([params.samtools_executable, 'sort',
                         ''.join([params.file_prefix, '_fixPG.bam']),
                         ''.join([params.file_prefix, '_fixPG_sorted'])])
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': samtools sort failed.', params.logfile)
    return_value = call([params.samtools_executable, 'index',
                         ''.join([params.file_prefix, '_fixPG_sorted.bam'])])
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': samtools index failed.', params.logfile)
    call(['rm', ''.join([params.file_prefix, '_fixPG.bam'])])
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Inserting @RG tag into header.', file=params.logfile)
    # Reheader the indexed _fixPG_sorted.bam to prepare for mutect
    picard_call = [params.java_executable, ''.join(['-Xmx', params.java_Xmx]),
                   '-jar'] #  Base java call
    picard_call.append(params.picard_jar)  # picard
    picard_call.append('AddOrReplaceReadGroups')  # module
    picard_call.append('CREATE_INDEX=true')
    picard_call.append(''.join(['I=', params.file_prefix, '_fixPG_sorted.bam']))
    picard_call.append(''.join(['O=', params.file_prefix,
                                '_fixPG_sorted_reheader.bam']))
    picard_call.append('SO=coordinate')
    picard_call.append('ID=1')
    picard_call.append(''.join(['LB=', params.file_prefix]))
    picard_call.append('PL=ILLUMINA')
    picard_call.append('PU=12345')
    picard_call.append(''.join(['SM=', params.sample_type]))
    with open(''.join([params.file_prefix, '_picard_log.txt']), 'w') as logfile:
        return_value = call(picard_call, stdout=logfile)
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': picard AddOrReplaceReadGroups failed.', params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': @RG ' +
          'inserted. Indexing bam', file=params.logfile)
    # Index _fixPG_sorted_reheader.bam file
    return_value = call([params.samtools_executable, 'index',
                         ''.join([params.file_prefix,
                                  '_fixPG_sorted_reheader.bam'])])
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': samtools index failed.', params.logfile)
    # Remove intermediate files
    call(['rm', ''.join([params.file_prefix, '_fixPG_sorted.bam']),
          ''.join([params.file_prefix, '_fixPG_sorted.bam.bai'])])

    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Alignment completed. Finishing up...', params.logfile)
    # Move files from temp directory to outdir
    prepare.move_output(params)
    print('RESULT ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': Process ' +
          'completed', file=params.logfile)
    params.logfile.close()
Example #6
0
def main():
    '''
    This wrapper script will run the entire alignment pipeline for genomic DNA
    (WGS or WXS) from alignment of fastqs, to sorting, indexing, and Read Group
    incorporation. The wrapper can even download and produce bwa references if
    required. The wrapper requires
    1. bwa (For aligning reads)
    2. java (For picard)
    3. picard tools (For read groups)
    4. samtools (For sam/bam manipulation)
    5. twoBitToFa from the kent tools library (For extracting the reference
            genome in case indexing is required)

    Unless specified, the program will look for default executables on $PATH.
    The program DOES NOT look for jar files and they are required to be
    passed during execution.
    '''
    #  Parse the arguments using prepare.parse_args()
    params = prepare.parse_args(main.__doc__, 'bwa', 'bwa_alignment')

    #  Params ERROR handling
    #  The memory option for java should be of the form Xmx10G or Xmx10M
    if not (params.java_Xmx.endswith('G') or params.java_Xmx.endswith('M')):
        raise pi_errors.ParameterError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': Please use a suitable value for --Xmx.', params.logfile)
    params.bwa_executable = pi_errors.test_param_value(params.bwa_executable,
                                                       'bwa', '--bwa',
                                                       params.logfile)
    params.samtools_executable = pi_errors.test_param_value(
        params.samtools_executable, 'samtools', '--samtools', params.logfile)
    params.java_executable = pi_errors.test_param_value(
        params.java_executable, 'java', '--java', params.logfile)
    #  If Indexing is required, does twoBitToFa point to a valid file?
    if params.index_location is None:
        params.tbtf_executable = pi_errors.test_param_value(
            params.tbtf_executable, 'twoBitToFa', '--twoBitToFa',
            params.logfile)
    if not params.picard_jar.endswith('jar'):
        raise pi_errors.ParameterError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': Please specify a valid jar file for picard!', params.logfile)
    else:
        params.picard_jar = pi_errors.test_param_value(params.picard_jar,
                                                       'picard',
                                                       '--picard_jar',
                                                       params.logfile)

    if params.RGID is None:
        params.RGID = params.file_prefix

    #read_group = ''.join(['\'@RG\\tID:', params.RGID, '\\tPL:ILLUMINA\\tSM:',
    #                      params.sample_type, '\''])
    # Check for indexes. If the user has specified that indexes need to
    # be created then do so.
    if params.index_location is None:
        print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
              'Indexing fasta...',
              file=params.logfile)
        if not os.path.exists(params.index_destination):
            prepare.py_mkdir(params.index_destination)
        index_path = params.index_destination
        genome_fasta = prepare.get_genome(params.genome_version, index_path,
                                          params.twoBitToFa_executable,
                                          params.logfile)
        print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ':    ' +
              'Running BWA index on fasta reference.',
              file=params.logfile)
        return_value = call([params.bwa_executable, 'index', genome_fasta])
        if return_value != 0:
            raise pi_errors.MyRuntimeError(
                dt.now().strftime('%I:%M %p %b %d, %Y') + \
                ': bwa index failed.', params.logfile)
        print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ':    ' +
              'Running samtools faidx.',
              file=params.logfile)
        return_value = call(
            [params.samtools_executable, 'faidx', genome_fasta])
        if return_value != 0:
            raise pi_errors.MyRuntimeError(
                dt.now().strftime('%I:%M %p %b %d, %Y') + \
                ': samtools faidx failed', params.logfile)
        index_prefix = genome_fasta
        print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
              'Indexing completed.',
              file=params.logfile)
    else:
        if params.index_location.endswith('.fa'):
            assert os.path.exists(
                params.index_location), 'Index file not found'
            index_prefix = params.index_location
        else:
            fastas = [
                x for x in os.listdir(params.index_location)
                if x.endswith(".fa")
            ]
            if len(fastas) == 1:
                index_prefix = "".join([params.index_location, '/', fastas[0]])
            elif len(fastas) == 0:
                raise pi_errors.InputFileError(
                    dt.now().strftime('%I:%M %p %b %d, %Y') + \
                    ': No valid fasta found in provided index folder',
                    params.logfile)
            else:
                raise pi_errors.InputFileError(
                    dt.now().strftime('%I:%M %p %b %d, %Y') + \
                    ':Multiple fastas found in provided index folder. Try ' + \
                    'running with --index_location /path/to/file/filename.fa',
                    params.logfile)

    # Move to working directory before doing I/O intensive alignment
    os.chdir(params.working_dir)

    # Align reads to sam file
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') +
          ': Aligning' + ' reads to reference.',
          file=params.logfile)
    bwa_call = [params.bwa_executable, 'mem']  # base call
    bwa_call.extend(['-t', str(params.n)])  # Number of threads
    #bwa_call.extend(['-R', read_group])  # Read group
    bwa_call.append(index_prefix)  # bwa index
    bwa_call.append(''.join(
        [params.file_path, '/', params.file_prefix, '_1.fastq']))
    bwa_call.append(''.join(
        [params.file_path, '/', params.file_prefix, '_2.fastq']))
    print(' '.join(bwa_call), file=params.logfile)
    with open(''.join([params.file_prefix, '.sam']), 'w') as samfile, \
            open(''.join([params.file_prefix, '_bwa_log.txt']), 'w') as logfile:
        return_value = call(bwa_call, stdout=samfile, stderr=logfile)
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': bwa mem failed.', params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Alignment completed. Converting to bam',
          file=params.logfile)
    # Convert the sam to a bam file
    with open(''.join([params.file_prefix, '.bam']), 'w') as bamfile:
        call([
            params.samtools_executable, 'view', '-bS', ''.join(
                [params.file_prefix, '.sam'])
        ],
             stdout=bamfile)
    call(['rm', ''.join([params.file_prefix, '.sam'])])
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') +
          ': bam file' +
          ' created. Preparing file for inserting RG into header.',
          file=params.logfile)
    # Fix PG line
    sam_header = check_output([
        params.samtools_executable, 'view', '-H',
        ''.join([params.file_prefix, '.bam'])
    ])
    sam_header = sam_header.strip().split(
        '\n')  # Strip whitespace and separate
    pg_line = sam_header[-1].split('\t')  # Grab @PG line + split by tab
    # Then remove the CL field form the PG line
    sam_header[-1] = '\t'.join([x for x in pg_line if not x.startswith('CL')])
    with open(''.join([params.file_prefix, '_sam.header']), 'w') as hdr_file:
        print('\n'.join(sam_header), file=hdr_file)
    with open(''.join([params.file_prefix, '_fixPG.bam']), 'w') as \
              fixpg_bamfile:
        return_value = call([
            params.samtools_executable, 'reheader', ''.join([
                params.file_prefix, '_sam.header'
            ]), ''.join([params.file_prefix, '.bam'])
        ],
                            stdout=fixpg_bamfile)
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': samtools reheader failed', params.logfile)
    call([
        'rm', ''.join([params.file_prefix, '.bam']),
        ''.join([params.file_prefix, '_sam.header'])
    ])
    # Sort and Index the _fixPG.bam file
    return_value = call([
        params.samtools_executable, 'sort',
        ''.join([params.file_prefix,
                 '_fixPG.bam']), ''.join([params.file_prefix, '_fixPG_sorted'])
    ])
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': samtools sort failed.', params.logfile)
    return_value = call([
        params.samtools_executable, 'index',
        ''.join([params.file_prefix, '_fixPG_sorted.bam'])
    ])
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': samtools index failed.', params.logfile)
    call(['rm', ''.join([params.file_prefix, '_fixPG.bam'])])
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Inserting @RG tag into header.',
          file=params.logfile)
    # Reheader the indexed _fixPG_sorted.bam to prepare for mutect
    picard_call = [
        params.java_executable, ''.join(['-Xmx', params.java_Xmx]), '-jar'
    ]  #  Base java call
    picard_call.append(params.picard_jar)  # picard
    picard_call.append('AddOrReplaceReadGroups')  # module
    picard_call.append('CREATE_INDEX=true')
    picard_call.append(''.join(['I=', params.file_prefix,
                                '_fixPG_sorted.bam']))
    picard_call.append(''.join(
        ['O=', params.file_prefix, '_fixPG_sorted_reheader.bam']))
    picard_call.append('SO=coordinate')
    picard_call.append('ID=1')
    picard_call.append(''.join(['LB=', params.file_prefix]))
    picard_call.append('PL=ILLUMINA')
    picard_call.append('PU=12345')
    picard_call.append(''.join(['SM=', params.sample_type]))
    with open(''.join([params.file_prefix, '_picard_log.txt']),
              'w') as logfile:
        return_value = call(picard_call, stdout=logfile)
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': picard AddOrReplaceReadGroups failed.', params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': @RG ' +
          'inserted. Indexing bam',
          file=params.logfile)
    # Index _fixPG_sorted_reheader.bam file
    return_value = call([
        params.samtools_executable, 'index',
        ''.join([params.file_prefix, '_fixPG_sorted_reheader.bam'])
    ])
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': samtools index failed.', params.logfile)
    # Remove intermediate files
    call([
        'rm', ''.join([params.file_prefix, '_fixPG_sorted.bam']),
        ''.join([params.file_prefix, '_fixPG_sorted.bam.bai'])
    ])

    print(
        'PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
        'Alignment completed. Finishing up...', params.logfile)
    # Move files from temp directory to outdir
    prepare.move_output(params)
    print('RESULT ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': Process ' +
          'completed',
          file=params.logfile)
    params.logfile.close()
def main():
    '''
    This wrapper script will run the tool PHLAT within the phlat docker
    container for the precision immuno project. The wrapper requires:
    1. PHLAT.py
    2. bowtie2
    3. gdown.pl (For donwloading the PHLAT Index - available from
       https://raw.githubusercontent.com/Nanolx/patchimage/master/tools/gdown.pl)

    Unless specified, the program will look for default executables on $PATH.
    The program DOES NOT look for jar files and they are required to be
    passed during execution.
    '''
    # Parse the arguments using prepare.parse_args()
    params = prepare.parse_args(main.__doc__, 'phlat', 'MHC_typing')

    # params ERROR handling
    if not params.phlat_executable.endswith('PHLAT.py'):
        params.phlat_executable = '/'.join([params.phlat_executable,
                                            'PHLAT.py'])
    params.phlat_executable = pi_errors.test_param_value(
        params.phlat_executable, 'PHLAT', '--phlat', params.logfile)
    params.bowtie2_executable = pi_errors.test_param_value(
        params.bowtie2_executable, 'bowtie2', '--bowtie2', params.logfile)
    phlat_dir = os.path.split(os.path.split(params.phlat_executable)[0])[0]
    params.gdownpl_executable = pi_errors.test_param_value(
        params.gdownpl_executable, 'gdown.pl', '--gdownpl', params.logfile)

    if params.index_location is None:
        print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') +': ' +
              'Downloading Indexes...', file=params.logfile)
        params.index_destination = os.path.abspath(params.index_destination)
        if not os.path.exists(params.index_destination):
            prepare.py_mkdir(params.index_destination)
        getindex_call = [params.gdownpl_executable, 'https://drive.google.com' +
                         '/uc?export=download&confirm=yAjx&id=0Bz-w5tutuZIYY3' +
                         'h5YlMzTjhnbGM', ''.join([params.index_destination,
                                                   '/index4phlat.tar.gz'])]
        print(getindex_call, file=params.logfile)
        return_value = call(getindex_call)
        if return_value != 0:
            raise pi_errors.MyRuntimeError(
                dt.now().strftime('%I:%M %p %b %d, %Y') + \
                ': Could not download indexes. Try manually downloading.',
                params.logfile)
        extract_call = ['tar', '-C', params.index_destination, '-zxvf',
                        '/'.join([params.index_destination,
                                  'index4phlat.tar.gz'])]
        return_value = call(extract_call)
        if return_value != 0:
            raise pi_errors.MyRuntimeError(
                dt.now().strftime('%I:%M %p %b %d, %Y') + \
                ': Index4phlat could not be extracted.', params.logfile)
        else:
            call(['rm', '/'.join([params.index_destination,
                                  'index4phlat.tar.gz'])])
        index_path = '/'.join([params.index_destination, 'index4phlat'])
        print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
              'Indexes Downloaded.', file=params.logfile)
    else:
        params.index_location = os.path.abspath(params.index_location)
        if not os.path.exists(''.join([params.index_location,
                                       '/ucsc.artHLA.1.bt2'])):
            raise pi_errors.InputFileError(
                dt.now().strftime('%I:%M %p %b %d, %Y') + \
                ': Index file not found.', params.logfile)
        else:
            index_path = params.index_location

    # Move to working directory before doing I/O intensive alignment
    os.chdir(params.working_dir)

    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') +':    ' +
          'Begining MHC Haplotyping', file=params.logfile)
    system_call = ['/usr/bin/env', 'python2.7', '-O', params.phlat_executable]
    system_call.extend(['-1', ''.join([params.file_path, '/',
                                       params.file_prefix, '_1.fastq'])]) # Fq1
    system_call.extend(['-2', ''.join([params.file_path, "/",
                                       params.file_prefix, '_2.fastq'])]) # Fq2
    system_call.extend(['-index', index_path]) # Index files
    system_call.extend(['-b2url', params.bowtie2_executable]) # Bowtie2
    system_call.extend(['-tag', ''.join([params.out_prefix])]) # DNA/RNA
    system_call.extend(['-e', phlat_dir]) # Phlat directory home
    system_call.extend(['-o', params.outdir]) # Output directory
    system_call.extend(['-p', str(params.n)]) # Number of threads

    # Call the program
    return_value = call(system_call)
    if return_value != 0:
        raise pi_errors.MyRuntimeError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': MHC Haplotyping failed.', params.logfile)
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': ' +
          'Alignment completed. Finishing up...', file=params.logfile)
    # Move files from temp directory to outdir
    prepare.move_output(params)
    print('RESULT ' + dt.now().strftime('%I:%M %p %b %d, %Y') + ': Process ' +
          'completed', file=params.logfile)
    params.logfile.close()
def process_parameters(params):
    '''
    This module conducts the error handling for all parmeters passed to the
    program.
    '''
    print('PROGRESS ' + dt.now().strftime('%I:%M %p %b %d, %Y') +
          ': Processing input parameters.', file=params.logfile)
    # Does the input vcf file exist?
    if not os.path.exists(''.join([params.file_path, '/', params.file_prefix,
                                   '.vcf'])):
        raise pi_errors.InputFileError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': Please provide a valid input file using --file_prefix',
            params.logfile)
    #  The memory option for java should be of the form Xmx10G or Xmx10M
    if not (params.java_Xmx.endswith('G') or params.java_Xmx.endswith('M')):
        raise pi_errors.ParameterError(
            dt.now().strftime('%I:%M %p %b %d, %Y') + \
            ': Please use a suitable value for --Xmx.', params.logfile)
    params.java_executable = pi_errors.test_param_value(params.java_executable,
                                                        'java',
                                                        '--java',
                                                        params.logfile)
    #  Does the provided snpeff binary provided exist?
    params.snpeff_jar = pi_errors.test_param_value(params.snpeff_jar,
                                                   'snpeff',
                                                   '--snpeff_jar',
                                                   params.logfile)
    params.use_snpeff_db = False
    #  Does the user want a snpEff packaged database?
    if params.config_file == 'PACKAGED':
        params.use_snpeff_db = True
        #  Has the snpeff reference to be used been provided?
        if params.reference_name == None:
            raise pi_errors.ParameterError(
                dt.now().strftime('%I:%M %p %b %d, %Y') + \
                ': --snp_reference is required if --config=PACKAGED.',
                params.logfile)
    # If a custom databse is desired, does it need to be created?
    if params.index_location is None:
        #  If the user has provided the location to the parent directory of data
        #  directory, make DATA_DIRECTORY point to data. If they have provided
        #  the link to data, make INDEX_DESTINATION point to the parent and
        #  DATA_DIRECTORY point to data.
        if os.path.split(params.index_destination.rstrip('/'))[1] != 'data':
            params.data_directory = '/'.join([params.index_destination, 'data'])
        else:
            params.data_directory = params.index_destination
            params.index_destination = \
                params.index_destination.rstrip('/').rstrip('/data')
        #  Create the data directory if needed
        if not os.path.exists(params.data_directory):
            prepare.py_mkdir(params.data_directory)
        #  If we're using a custom databse, thre is nothing more to do
        if params.use_snpeff_db:
            return None
        #  Initialise the reference name
        params.reference_name = ''.join([params.genome_version, '_custom'])
        # make a variable to gold GENOME_VERSION_custom
        genome_folder = '/'.join([params.data_directory, params.reference_name])
        prepare.py_mkdir(genome_folder)
        #  If the genome fasta isn't provided or is provided a wrong value,
        #  download it
        if params.genome_fasta == 'DOWNLOAD' or not \
                os.path.exists(params.genome_fasta):
            #  Does the provided tbtf binary point to a valid file?
            params.tbtf_executable = pi_errors.test_param_value(
                params.tbtf_executable, 'twoBitToFa', '--twoBitToFa',
                params.logfile)
            params.genome_fasta = prepare.get_genome(
                params.genome_version, genome_folder,
                params.tbtf_executable, params.logfile)
            #  Rename genome fasta
            call(['mv', params.genome_fasta, '/'.join([genome_folder,
                                                       'sequences.fa'])])
        else:
            params.genome_fasta = os.path.abspath(params.genome_fasta)
            #  Link sequencesfa to genome fasta
            call(['ln', '-s', '-T', params.genome_fasta,
                  '/'.join([genome_folder, 'sequences.fa'])])
        #  Download the gencode GTF file
        params.gtf_file = prepare.get_gtf(params.genome_version,
                                          genome_folder,
                                          params.logfile)
        #  Rename gtf file
        call(['mv', params.gtf_file, '/'.join([genome_folder, 'genes.gtf'])])
    # If it has been provided, set up the config file
    else:
        #  If the user has provided the location to the parent directory of data
        #  directory, make DATA_DIRECTORY point to data. If they have provided
        #  the link to data, make INDEX_LOCATION point to the parent and
        #  DATA_DIRECTORY point to data.
        if os.path.split(params.index_location.rstrip('/'))[1] != 'data':
            params.data_directory = '/'.join([params.index_location, 'data'])
        else:
            params.data_directory = params.index_location
            params.index_location = \
                params.index_location.rstrip('/').rstrip('/data')
        #  If we're using a custom databse, thre is nothing more to do
        if params.use_snpeff_db:
            return None
        #  If the config file hasn't been provided, is it in INDEX_LOCATION
        #  AND does GENOME_VERSION_custom exist (i.e. was it created by
        #  this script?)
        if params.config_file is None:
            params.config_file = pi_errors.test_param_value(
                '/'.join([params.index_location, 'snpEff.config']),
                'snpEff.config', '--config', params.logfile)
            #  Dummy variable to ensure the GENOME_VERSION_custom exists
            _ = pi_errors.test_param_value(
                ''.join([params.data_directory, '/', params.genome_version,
                         '_custom']), '_'.join([params.genome_version, 'custom'
                                               ]), '--snpeff_reference and' +
                '--config', params.logfile)
            params.reference_name = '_'.join([params.genome_version, 'custom'])
        #  If a config file has been provided, does it point to a legit file and
        #  has the reference name also been provided?
        else:
            params.config_file = pi_errors.test_param_value(
                params.config_file, 'snpEff config file', '--config',
                params.logfile)
            if params.reference_name is None:
                raise pi_errors.ParameterError(
                    dt.now().strftime('%I:%M %p %b %d, %Y') + \
                    ': --snpeff_reference is required if --config points to' + \
                    ' a custom file.', params.logfile)
    return None