def create_plots(ionstats_filepath, flows):
try:
graph_max_x = int(50 * math.ceil(0.014 * int(flows)))
except:
graph_max_x = 800
# Use ionstats alignment results to generate plots
ionstats_plots.alignment_rate_plot2(ionstats_filepath,
'alignment_rate_plot.png', graph_max_x)
ionstats_plots.base_error_plot(ionstats_filepath, 'base_error_plot.png',
graph_max_x)
ionstats_plots.old_aq_length_histogram(ionstats_filepath,
'Filtered_Alignments_Q10.png',
'AQ10', 'red')
ionstats_plots.old_aq_length_histogram(ionstats_filepath,
'Filtered_Alignments_Q17.png',
'AQ17', 'yellow')
ionstats_plots.old_aq_length_histogram(ionstats_filepath,
'Filtered_Alignments_Q20.png',
'AQ20', 'green')
ionstats_plots.old_aq_length_histogram(ionstats_filepath,
'Filtered_Alignments_Q47.png',
'AQ47', 'purple')
def create_plots(ionstats_filepath,flows):
try:
graph_max_x = int(50 * math.ceil(0.014 * int(flows)))
except:
graph_max_x = 800
# Use ionstats alignment results to generate plots
ionstats_plots.alignment_rate_plot2(ionstats_filepath,'alignment_rate_plot.png', graph_max_x)
ionstats_plots.base_error_plot(ionstats_filepath,'base_error_plot.png', graph_max_x)
ionstats_plots.old_aq_length_histogram(ionstats_filepath,'Filtered_Alignments_Q10.png', 'AQ10', 'red')
ionstats_plots.old_aq_length_histogram(ionstats_filepath,'Filtered_Alignments_Q17.png', 'AQ17', 'yellow')
ionstats_plots.old_aq_length_histogram(ionstats_filepath,'Filtered_Alignments_Q20.png', 'AQ20', 'green')
ionstats_plots.old_aq_length_histogram(ionstats_filepath,'Filtered_Alignments_Q47.png', 'AQ47', 'purple')
if args.merge_plots:
printtime("Generating plots for merged report")
ionstats_file = 'ionstats_alignment.json'
try:
stats = json.load(open(ionstats_file))
l = stats['full']['max_read_length']
graph_max_x = int(round(l + 49, -2))
# Make alignment_rate_plot.png and base_error_plot.png
ionstats_plots.alignment_rate_plot2(ionstats_file,
'alignment_rate_plot.png',
int(graph_max_x))
ionstats_plots.base_error_plot(ionstats_file,
'base_error_plot.png',
int(graph_max_x))
except:
traceback.print_exc()
if args.zip and len(args.files) > 1:
# zip barcoded files
zipname = args.zip
printtime("Zip merged barcode files to %s" % zipname)
for filename in args.files:
if os.path.exists(filename):
try:
make_zip(zipname, filename, arcname=filename)
except:
print("ERROR: zip target: %s" % filename)
traceback.print_exc()
# Plot higher detail barcode specific histogram
ionstats_plots.old_read_length_histogram(
os.path.join(ionstats_folder, dataset['file_prefix']+'.'+ionstats_file),
os.path.join(env['BASECALLER_RESULTS'], dataset['file_prefix']+'.read_len_histogram.png'),
graph_max_x)
except:
traceback.print_exc()
if reference_selected:
try:
# Use ionstats alignment results to generate plots
ionstats_plots.alignment_rate_plot2('ionstats_alignment.json','alignment_rate_plot.png', graph_max_x)
ionstats_plots.base_error_plot('ionstats_alignment.json','base_error_plot.png', graph_max_x)
ionstats_plots.old_aq_length_histogram('ionstats_alignment.json','Filtered_Alignments_Q10.png', 'AQ10', 'red')
ionstats_plots.old_aq_length_histogram('ionstats_alignment.json','Filtered_Alignments_Q17.png', 'AQ17', 'yellow')
ionstats_plots.old_aq_length_histogram('ionstats_alignment.json','Filtered_Alignments_Q20.png', 'AQ20', 'green')
ionstats_plots.old_aq_length_histogram('ionstats_alignment.json','Filtered_Alignments_Q47.png', 'AQ47', 'purple')
except:
traceback.print_exc()
try:
wells_beadogram.generate_wells_beadogram(env['BASECALLER_RESULTS'], env['SIGPROC_RESULTS'])
except:
printtime ("ERROR: Wells beadogram generation failed")
traceback.print_exc()
set_result_status('TF Processing')
error_summary_file = bamfile.split('.bam')[0] + '.ionstats_error_summary.h5'
generate_ionstats(bamfile, ionstats_file, error_summary_file, graph_max_x)
if args.merge_plots:
printtime("Generating plots for merged report")
ionstats_file = 'ionstats_alignment.json'
try:
stats = json.load(open(ionstats_file))
l = stats['full']['max_read_length']
graph_max_x = int(round(l + 49, -2))
# Make alignment_rate_plot.png and base_error_plot.png
ionstats_plots.alignment_rate_plot2(ionstats_file, 'alignment_rate_plot.png', int(graph_max_x))
ionstats_plots.base_error_plot(ionstats_file, 'base_error_plot.png', int(graph_max_x))
except:
traceback.print_exc()
if args.zip and len(args.files) > 1:
# zip barcoded files
zipname = args.zip
printtime("Zip merged barcode files to %s" % zipname)
for filename in args.files:
if os.path.exists(filename):
try:
make_zip(zipname, filename, arcname=filename)
except:
print("ERROR: zip target: %s" % filename)
traceback.print_exc()
dataset["file_prefix"] + ".read_len_histogram.png",
),
graph_max_x,
)
except Exception:
traceback.print_exc()
if reference_selected:
try:
# Use ionstats alignment results to generate plots
ionstats_plots.alignment_rate_plot2("ionstats_alignment.json",
"alignment_rate_plot.png",
graph_max_x)
ionstats_plots.base_error_plot("ionstats_alignment.json",
"base_error_plot.png",
graph_max_x)
ionstats_plots.old_aq_length_histogram(
"ionstats_alignment.json",
"Filtered_Alignments_Q10.png",
"AQ10",
"red",
)
ionstats_plots.old_aq_length_histogram(
"ionstats_alignment.json",
"Filtered_Alignments_Q17.png",
"AQ17",
"yellow",
)
ionstats_plots.old_aq_length_histogram(
"ionstats_alignment.json",
# Plot higher detail barcode specific histogram
ionstats_plots.old_read_length_histogram(
os.path.join(ionstats_folder, dataset['file_prefix']+'.'+ionstats_file),
os.path.join(env['BASECALLER_RESULTS'], dataset['file_prefix']+'.read_len_histogram.png'),
graph_max_x)
except:
traceback.print_exc()
if reference_selected:
try:
# Use ionstats alignment results to generate plots
ionstats_plots.alignment_rate_plot2(
'ionstats_alignment.json', 'alignment_rate_plot.png', graph_max_x)
ionstats_plots.base_error_plot('ionstats_alignment.json', 'base_error_plot.png', graph_max_x)
ionstats_plots.old_aq_length_histogram(
'ionstats_alignment.json', 'Filtered_Alignments_Q10.png', 'AQ10', 'red')
ionstats_plots.old_aq_length_histogram(
'ionstats_alignment.json', 'Filtered_Alignments_Q17.png', 'AQ17', 'yellow')
ionstats_plots.old_aq_length_histogram(
'ionstats_alignment.json', 'Filtered_Alignments_Q20.png', 'AQ20', 'green')
ionstats_plots.old_aq_length_histogram(
'ionstats_alignment.json', 'Filtered_Alignments_Q47.png', 'AQ47', 'purple')
except:
traceback.print_exc()
try:
wells_beadogram.generate_wells_beadogram(env['BASECALLER_RESULTS'], env['SIGPROC_RESULTS'])
except:
printtime("ERROR: Wells beadogram generation failed")
def merge_alignment_stats(dirs, BASECALLER_RESULTS, ALIGNMENT_RESULTS, flows):
datasets_basecaller = {}
try:
f = open(os.path.join(BASECALLER_RESULTS,"datasets_basecaller.json"),'r')
datasets_basecaller = json.load(f)
f.close()
except:
printtime("ERROR: problem parsing %s" % os.path.join(BASECALLER_RESULTS,"datasets_basecaller.json"))
traceback.print_exc()
return
try:
graph_max_x = int(50 * math.ceil(0.014 * int(flows)))
except:
graph_max_x = 800
########################################################
# Merge ionstats_alignment.json
# First across blocks, then across barcoded
########################################################
try:
composite_filename_list = []
for dataset in datasets_basecaller["datasets"]:
composite_filename = os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.ionstats_alignment.json')
barcode_filename_list = [os.path.join(dir,ALIGNMENT_RESULTS,dataset['file_prefix']+'.ionstats_alignment.json') for dir in dirs]
barcode_filename_list = [filename for filename in barcode_filename_list if os.path.exists(filename)]
ionstats.reduce_stats(barcode_filename_list,composite_filename)
if os.path.exists(composite_filename):
composite_filename_list.append(composite_filename)
ionstats.reduce_stats(composite_filename_list,os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'))
except:
printtime("ERROR: Failed to merge ionstats_alignment.json")
traceback.print_exc()
# Use ionstats alignment results to generate plots
ionstats_plots.alignment_rate_plot2(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'alignment_rate_plot.png', graph_max_x)
ionstats_plots.base_error_plot(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'base_error_plot.png', graph_max_x)
ionstats_plots.old_aq_length_histogram(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'Filtered_Alignments_Q10.png', 'AQ10', 'red')
ionstats_plots.old_aq_length_histogram(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'Filtered_Alignments_Q17.png', 'AQ17', 'yellow')
ionstats_plots.old_aq_length_histogram(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'Filtered_Alignments_Q20.png', 'AQ20', 'green')
ionstats_plots.old_aq_length_histogram(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'Filtered_Alignments_Q47.png', 'AQ47', 'purple')
# Generate alignment_barcode_summary.csv
barcodelist_path = 'barcodeList.txt'
if not os.path.exists(barcodelist_path):
barcodelist_path = '../barcodeList.txt'
if not os.path.exists(barcodelist_path):
barcodelist_path = '../../barcodeList.txt'
if not os.path.exists(barcodelist_path):
barcodelist_path = '../../../barcodeList.txt'
if not os.path.exists(barcodelist_path):
barcodelist_path = '../../../../barcodeList.txt'
if os.path.exists(barcodelist_path):
printtime("Barcode processing, aggregate")
aggregate_alignment ("./",barcodelist_path)
def alignment_post_processing(
libraryName,
BASECALLER_RESULTS,
ALIGNMENT_RESULTS,
flows,
mark_duplicates):
datasets_basecaller = {}
try:
f = open(os.path.join(BASECALLER_RESULTS,"datasets_basecaller.json"),'r')
datasets_basecaller = json.load(f)
f.close()
except:
printtime("ERROR: problem parsing %s" % os.path.join(BASECALLER_RESULTS,"datasets_basecaller.json"))
traceback.print_exc()
return
try:
graph_max_x = int(50 * math.ceil(0.014 * int(flows)))
except:
graph_max_x = 800
alignment_file_list = []
for dataset in datasets_basecaller["datasets"]:
if not os.path.exists(os.path.join(BASECALLER_RESULTS, dataset['basecaller_bam'])):
continue
ionstats.generate_ionstats_alignment(
os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.bam'),
os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.ionstats_alignment.json'),
graph_max_x)
ionstats2alignstats(libraryName,
os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.ionstats_alignment.json'),
os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.alignment.summary'))
alignment_file_list.append(os.path.join(ALIGNMENT_RESULTS, dataset['file_prefix']+'.ionstats_alignment.json'))
# In Progress: merge ionstats alignment results
ionstats.reduce_stats(alignment_file_list,os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'))
ionstats2alignstats(libraryName,
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
os.path.join(ALIGNMENT_RESULTS,'alignment.summary'))
# Special legacy post-processing.
# Generate merged rawlib.bam on barcoded runs
composite_bam_filename = os.path.join(ALIGNMENT_RESULTS,'rawlib.bam')
if not os.path.exists(composite_bam_filename):
bam_file_list = []
for dataset in datasets_basecaller["datasets"]:
bam_name = os.path.join(ALIGNMENT_RESULTS,os.path.basename(dataset['file_prefix'])+'.bam')
if os.path.exists(bam_name):
bam_file_list.append(bam_name)
blockprocessing.merge_bam_files(bam_file_list,composite_bam_filename,composite_bam_filename+'.bai',mark_duplicates)
# Generate alignment_barcode_summary.csv
#TODO: use datasets_basecaller.json + *.ionstats_alignment.json instead of barcodeList.txt and alignment_*.summary
barcodelist_path = 'barcodeList.txt'
if not os.path.exists(barcodelist_path):
barcodelist_path = '../barcodeList.txt'
if not os.path.exists(barcodelist_path):
barcodelist_path = '../../barcodeList.txt'
if not os.path.exists(barcodelist_path):
barcodelist_path = '../../../barcodeList.txt'
if not os.path.exists(barcodelist_path):
barcodelist_path = '../../../../barcodeList.txt'
if os.path.exists(barcodelist_path):
printtime("Barcode processing, aggregate")
aggregate_alignment ("./",barcodelist_path)
# These graphs are likely obsolete
#makeAlignGraphs()
# In Progress: Use ionstats alignment results to generate plots
ionstats_plots.alignment_rate_plot2(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'alignment_rate_plot.png', graph_max_x)
ionstats_plots.base_error_plot(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'base_error_plot.png', graph_max_x)
ionstats_plots.old_aq_length_histogram(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'Filtered_Alignments_Q10.png', 'AQ10', 'red')
ionstats_plots.old_aq_length_histogram(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'Filtered_Alignments_Q17.png', 'AQ17', 'yellow')
ionstats_plots.old_aq_length_histogram(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'Filtered_Alignments_Q20.png', 'AQ20', 'green')
ionstats_plots.old_aq_length_histogram(
os.path.join(ALIGNMENT_RESULTS,'ionstats_alignment.json'),
'Filtered_Alignments_Q47.png', 'AQ47', 'purple')