Example #1
0
def create_plots(ionstats_filepath, flows):

    try:
        graph_max_x = int(50 * math.ceil(0.014 * int(flows)))
    except:
        graph_max_x = 800

    # Use ionstats alignment results to generate plots
    ionstats_plots.alignment_rate_plot2(ionstats_filepath,
                                        'alignment_rate_plot.png', graph_max_x)

    ionstats_plots.base_error_plot(ionstats_filepath, 'base_error_plot.png',
                                   graph_max_x)

    ionstats_plots.old_aq_length_histogram(ionstats_filepath,
                                           'Filtered_Alignments_Q10.png',
                                           'AQ10', 'red')
    ionstats_plots.old_aq_length_histogram(ionstats_filepath,
                                           'Filtered_Alignments_Q17.png',
                                           'AQ17', 'yellow')
    ionstats_plots.old_aq_length_histogram(ionstats_filepath,
                                           'Filtered_Alignments_Q20.png',
                                           'AQ20', 'green')
    ionstats_plots.old_aq_length_histogram(ionstats_filepath,
                                           'Filtered_Alignments_Q47.png',
                                           'AQ47', 'purple')
Example #2
0
def create_plots(ionstats_filepath,flows):

    try:
        graph_max_x = int(50 * math.ceil(0.014 * int(flows)))
    except:
        graph_max_x = 800

    # Use ionstats alignment results to generate plots
    ionstats_plots.alignment_rate_plot2(ionstats_filepath,'alignment_rate_plot.png', graph_max_x)

    ionstats_plots.base_error_plot(ionstats_filepath,'base_error_plot.png', graph_max_x)

    ionstats_plots.old_aq_length_histogram(ionstats_filepath,'Filtered_Alignments_Q10.png', 'AQ10', 'red')
    ionstats_plots.old_aq_length_histogram(ionstats_filepath,'Filtered_Alignments_Q17.png', 'AQ17', 'yellow')
    ionstats_plots.old_aq_length_histogram(ionstats_filepath,'Filtered_Alignments_Q20.png', 'AQ20', 'green')
    ionstats_plots.old_aq_length_histogram(ionstats_filepath,'Filtered_Alignments_Q47.png', 'AQ47', 'purple')
Example #3
0
            generate_ionstats(bamfile, ionstats_file, error_summary_file,
                              graph_max_x)

    if args.merge_plots:
        printtime("Generating plots for merged report")
        ionstats_file = 'ionstats_alignment.json'

        try:
            stats = json.load(open(ionstats_file))
            l = stats['full']['max_read_length']
            graph_max_x = int(round(l + 49, -2))

            # Make alignment_rate_plot.png and base_error_plot.png
            ionstats_plots.alignment_rate_plot2(ionstats_file,
                                                'alignment_rate_plot.png',
                                                int(graph_max_x))
            ionstats_plots.base_error_plot(ionstats_file,
                                           'base_error_plot.png',
                                           int(graph_max_x))
        except:
            traceback.print_exc()

    if args.zip and len(args.files) > 1:
        # zip barcoded files
        zipname = args.zip
        printtime("Zip merged barcode files to %s" % zipname)
        for filename in args.files:
            if os.path.exists(filename):
                try:
                    make_zip(zipname, filename, arcname=filename)
Example #4
0
                    graph_max_x)

                # Plot higher detail barcode specific histogram
                ionstats_plots.old_read_length_histogram(
                    os.path.join(ionstats_folder, dataset['file_prefix']+'.'+ionstats_file),
                    os.path.join(env['BASECALLER_RESULTS'], dataset['file_prefix']+'.read_len_histogram.png'),
                    graph_max_x)


        except:
            traceback.print_exc()

        if reference_selected:
            try:
                # Use ionstats alignment results to generate plots
                ionstats_plots.alignment_rate_plot2('ionstats_alignment.json','alignment_rate_plot.png', graph_max_x)
                ionstats_plots.base_error_plot('ionstats_alignment.json','base_error_plot.png', graph_max_x)
                ionstats_plots.old_aq_length_histogram('ionstats_alignment.json','Filtered_Alignments_Q10.png', 'AQ10', 'red')
                ionstats_plots.old_aq_length_histogram('ionstats_alignment.json','Filtered_Alignments_Q17.png', 'AQ17', 'yellow')
                ionstats_plots.old_aq_length_histogram('ionstats_alignment.json','Filtered_Alignments_Q20.png', 'AQ20', 'green')
                ionstats_plots.old_aq_length_histogram('ionstats_alignment.json','Filtered_Alignments_Q47.png', 'AQ47', 'purple')
            except:
                traceback.print_exc()

        try:
            wells_beadogram.generate_wells_beadogram(env['BASECALLER_RESULTS'], env['SIGPROC_RESULTS'])
        except:
            printtime ("ERROR: Wells beadogram generation failed")
            traceback.print_exc()

        set_result_status('TF Processing')
Example #5
0
                ionstats_file = bamfile.split('.bam')[0] + '.ionstats_alignment.json'
                error_summary_file = bamfile.split('.bam')[0] + '.ionstats_error_summary.h5'

            generate_ionstats(bamfile, ionstats_file, error_summary_file, graph_max_x)

    if args.merge_plots:
        printtime("Generating plots for merged report")
        ionstats_file = 'ionstats_alignment.json'

        try:
            stats = json.load(open(ionstats_file))
            l = stats['full']['max_read_length']
            graph_max_x = int(round(l + 49, -2))

            # Make alignment_rate_plot.png and base_error_plot.png
            ionstats_plots.alignment_rate_plot2(ionstats_file, 'alignment_rate_plot.png', int(graph_max_x))
            ionstats_plots.base_error_plot(ionstats_file, 'base_error_plot.png', int(graph_max_x))
        except:
            traceback.print_exc()

    if args.zip and len(args.files) > 1:
        # zip barcoded files
        zipname = args.zip
        printtime("Zip merged barcode files to %s" % zipname)
        for filename in args.files:
            if os.path.exists(filename):
                try:
                    make_zip(zipname, filename, arcname=filename)
                except:
                    print("ERROR: zip target: %s" % filename)
                    traceback.print_exc()
Example #6
0
                                 dataset["file_prefix"] + "." + ionstats_file),
                    os.path.join(
                        env["BASECALLER_RESULTS"],
                        dataset["file_prefix"] + ".read_len_histogram.png",
                    ),
                    graph_max_x,
                )

        except Exception:
            traceback.print_exc()

        if reference_selected:
            try:
                # Use ionstats alignment results to generate plots
                ionstats_plots.alignment_rate_plot2("ionstats_alignment.json",
                                                    "alignment_rate_plot.png",
                                                    graph_max_x)
                ionstats_plots.base_error_plot("ionstats_alignment.json",
                                               "base_error_plot.png",
                                               graph_max_x)
                ionstats_plots.old_aq_length_histogram(
                    "ionstats_alignment.json",
                    "Filtered_Alignments_Q10.png",
                    "AQ10",
                    "red",
                )
                ionstats_plots.old_aq_length_histogram(
                    "ionstats_alignment.json",
                    "Filtered_Alignments_Q17.png",
                    "AQ17",
                    "yellow",
Example #7
0
                    os.path.join(env['BASECALLER_RESULTS'], dataset['file_prefix']+'.sparkline.png'),
                    graph_max_x)

                # Plot higher detail barcode specific histogram
                ionstats_plots.old_read_length_histogram(
                    os.path.join(ionstats_folder, dataset['file_prefix']+'.'+ionstats_file),
                    os.path.join(env['BASECALLER_RESULTS'], dataset['file_prefix']+'.read_len_histogram.png'),
                    graph_max_x)

        except:
            traceback.print_exc()

        if reference_selected:
            try:
                # Use ionstats alignment results to generate plots
                ionstats_plots.alignment_rate_plot2(
                    'ionstats_alignment.json', 'alignment_rate_plot.png', graph_max_x)
                ionstats_plots.base_error_plot('ionstats_alignment.json', 'base_error_plot.png', graph_max_x)
                ionstats_plots.old_aq_length_histogram(
                    'ionstats_alignment.json', 'Filtered_Alignments_Q10.png', 'AQ10', 'red')
                ionstats_plots.old_aq_length_histogram(
                    'ionstats_alignment.json', 'Filtered_Alignments_Q17.png', 'AQ17', 'yellow')
                ionstats_plots.old_aq_length_histogram(
                    'ionstats_alignment.json', 'Filtered_Alignments_Q20.png', 'AQ20', 'green')
                ionstats_plots.old_aq_length_histogram(
                    'ionstats_alignment.json', 'Filtered_Alignments_Q47.png', 'AQ47', 'purple')
            except:
                traceback.print_exc()

        try:
            wells_beadogram.generate_wells_beadogram(env['BASECALLER_RESULTS'], env['SIGPROC_RESULTS'])
        except:
Example #8
0
def merge_alignment_stats(dirs, BASECALLER_RESULTS, ALIGNMENT_RESULTS, flows):

    datasets_basecaller = {}
    try:
        f = open(os.path.join(BASECALLER_RESULTS,"datasets_basecaller.json"),'r')
        datasets_basecaller = json.load(f)
        f.close()
    except:
        printtime("ERROR: problem parsing %s" % os.path.join(BASECALLER_RESULTS,"datasets_basecaller.json"))
        traceback.print_exc()
        return



    try:
        graph_max_x = int(50 * math.ceil(0.014 * int(flows)))
    except:
        graph_max_x = 800




    ########################################################
    # Merge ionstats_alignment.json
    # First across blocks, then across barcoded
    ########################################################

    try:
        composite_filename_list = []
        for dataset in datasets_basecaller["datasets"]:
            composite_filename = os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.ionstats_alignment.json')
            barcode_filename_list = [os.path.join(dir,ALIGNMENT_RESULTS,dataset['file_prefix']+'.ionstats_alignment.json') for dir in dirs]
            barcode_filename_list = [filename for filename in barcode_filename_list if os.path.exists(filename)]
            ionstats.reduce_stats(barcode_filename_list,composite_filename)
            if os.path.exists(composite_filename):
                composite_filename_list.append(composite_filename)

        ionstats.reduce_stats(composite_filename_list,os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'))
    except:
        printtime("ERROR: Failed to merge ionstats_alignment.json")
        traceback.print_exc()

    # Use ionstats alignment results to generate plots
    ionstats_plots.alignment_rate_plot2(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
            'alignment_rate_plot.png', graph_max_x)
    ionstats_plots.base_error_plot(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
            'base_error_plot.png', graph_max_x)
    ionstats_plots.old_aq_length_histogram(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
            'Filtered_Alignments_Q10.png', 'AQ10', 'red')
    ionstats_plots.old_aq_length_histogram(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
            'Filtered_Alignments_Q17.png', 'AQ17', 'yellow')
    ionstats_plots.old_aq_length_histogram(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
            'Filtered_Alignments_Q20.png', 'AQ20', 'green')
    ionstats_plots.old_aq_length_histogram(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'), 
            'Filtered_Alignments_Q47.png', 'AQ47', 'purple')



    # Generate alignment_barcode_summary.csv
    barcodelist_path = 'barcodeList.txt'
    if not os.path.exists(barcodelist_path):
        barcodelist_path = '../barcodeList.txt'
    if not os.path.exists(barcodelist_path):
        barcodelist_path = '../../barcodeList.txt'
    if not os.path.exists(barcodelist_path):
        barcodelist_path = '../../../barcodeList.txt'
    if not os.path.exists(barcodelist_path):
        barcodelist_path = '../../../../barcodeList.txt'
    if os.path.exists(barcodelist_path):
        printtime("Barcode processing, aggregate")
        aggregate_alignment ("./",barcodelist_path)
Example #9
0
def alignment_post_processing(
        libraryName,
        BASECALLER_RESULTS,
        ALIGNMENT_RESULTS,
        flows,
        mark_duplicates):


    datasets_basecaller = {}
    try:
        f = open(os.path.join(BASECALLER_RESULTS,"datasets_basecaller.json"),'r')
        datasets_basecaller = json.load(f)
        f.close()
    except:
        printtime("ERROR: problem parsing %s" % os.path.join(BASECALLER_RESULTS,"datasets_basecaller.json"))
        traceback.print_exc()
        return

    try:
        graph_max_x = int(50 * math.ceil(0.014 * int(flows)))
    except:
        graph_max_x = 800



    alignment_file_list = []

    for dataset in datasets_basecaller["datasets"]:
        if not os.path.exists(os.path.join(BASECALLER_RESULTS, dataset['basecaller_bam'])):
            continue

        ionstats.generate_ionstats_alignment(
                os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.bam'),
                os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.ionstats_alignment.json'),
                graph_max_x)
        ionstats2alignstats(libraryName,
                os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.ionstats_alignment.json'),
                os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.alignment.summary'))

        alignment_file_list.append(os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.ionstats_alignment.json'))

    # In Progress: merge ionstats alignment results
    ionstats.reduce_stats(alignment_file_list,os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'))    
    ionstats2alignstats(libraryName,
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
            os.path.join(ALIGNMENT_RESULTS,'alignment.summary'))

    # Special legacy post-processing.
    # Generate merged rawlib.bam on barcoded runs

    composite_bam_filename = os.path.join(ALIGNMENT_RESULTS,'rawlib.bam')
    if not os.path.exists(composite_bam_filename):

        bam_file_list = []
        for dataset in datasets_basecaller["datasets"]:
            bam_name = os.path.join(ALIGNMENT_RESULTS,os.path.basename(dataset['file_prefix'])+'.bam')
            if os.path.exists(bam_name):
                bam_file_list.append(bam_name)

        blockprocessing.merge_bam_files(bam_file_list,composite_bam_filename,composite_bam_filename+'.bai',mark_duplicates)

    # Generate alignment_barcode_summary.csv
    #TODO: use datasets_basecaller.json + *.ionstats_alignment.json instead of barcodeList.txt and alignment_*.summary
    barcodelist_path = 'barcodeList.txt'
    if not os.path.exists(barcodelist_path):
        barcodelist_path = '../barcodeList.txt'
    if not os.path.exists(barcodelist_path):
        barcodelist_path = '../../barcodeList.txt'
    if not os.path.exists(barcodelist_path):
        barcodelist_path = '../../../barcodeList.txt'
    if not os.path.exists(barcodelist_path):
        barcodelist_path = '../../../../barcodeList.txt'
    if os.path.exists(barcodelist_path):
        printtime("Barcode processing, aggregate")
        aggregate_alignment ("./",barcodelist_path)

    # These graphs are likely obsolete
    #makeAlignGraphs()

    # In Progress: Use ionstats alignment results to generate plots
    ionstats_plots.alignment_rate_plot2(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
            'alignment_rate_plot.png', graph_max_x)
    ionstats_plots.base_error_plot(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
            'base_error_plot.png', graph_max_x)
    ionstats_plots.old_aq_length_histogram(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
            'Filtered_Alignments_Q10.png', 'AQ10', 'red')
    ionstats_plots.old_aq_length_histogram(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
            'Filtered_Alignments_Q17.png', 'AQ17', 'yellow')
    ionstats_plots.old_aq_length_histogram(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
            'Filtered_Alignments_Q20.png', 'AQ20', 'green')
    ionstats_plots.old_aq_length_histogram(
            os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'), 
            'Filtered_Alignments_Q47.png', 'AQ47', 'purple')