def main():
def print_stacktrace_if_debug():
debug_flag = False
if 'args' in vars() and 'debug' in args:
debug_flag = args.debug
if debug_flag:
traceback.print_exc(file=sys.stdout)
error(traceback.format_exc())
try:
description = ['~~~CRISPRessoWGS~~~','-Analysis of CRISPR/Cas9 outcomes from WGS data-']
wgs_string = r'''
____________
| __ __ |
|| |/ _ (_ |
||/\|\__)__) |
|____________|
'''
print(CRISPRessoShared.get_crispresso_header(description,wgs_string))
parser = CRISPRessoShared.getCRISPRessoArgParser(parserTitle = 'CRISPRessoWGS Parameters',requiredParams={})
#tool specific optional
parser.add_argument('-b','--bam_file', type=str, help='WGS aligned bam file', required=True,default='bam filename' )
parser.add_argument('-f','--region_file', type=str, help='Regions description file. A BED format file containing the regions to analyze, one per line. The REQUIRED\
columns are: chr_id(chromosome name), bpstart(start position), bpend(end position), the optional columns are:name (an unique indentifier for the region), guide_seq, expected_hdr_amplicon_seq,coding_seq, see CRISPResso help for more details on these last 3 parameters)', required=True)
parser.add_argument('-r','--reference_file', type=str, help='A FASTA format reference file (for example hg19.fa for the human genome)', default='',required=True)
parser.add_argument('--min_reads_to_use_region', type=float, help='Minimum number of reads that align to a region to perform the CRISPResso analysis', default=10)
parser.add_argument('--skip_failed', help='Continue with pooled analysis even if one sample fails',action='store_true')
parser.add_argument('--gene_annotations', type=str, help='Gene Annotation Table from UCSC Genome Browser Tables (http://genome.ucsc.edu/cgi-bin/hgTables?command=start), \
please select as table "knowGene", as output format "all fields from selected table" and as file returned "gzip compressed"', default='')
parser.add_argument('-p','--n_processes',type=int, help='Specify the number of processes to use for the quantification.\
Please use with caution since increasing this parameter will increase the memory required to run CRISPResso.',default=1)
parser.add_argument('--crispresso_command', help='CRISPResso command to call',default='CRISPResso')
args = parser.parse_args()
crispresso_options = CRISPRessoShared.get_crispresso_options()
options_to_ignore = set(['fastq_r1','fastq_r2','amplicon_seq','amplicon_name','output_folder','name'])
crispresso_options_for_wgs = list(crispresso_options-options_to_ignore)
info('Checking dependencies...')
if check_samtools() and check_bowtie2():
info('\n All the required dependencies are present!')
else:
sys.exit(1)
#check files
check_file(args.bam_file)
check_file(args.reference_file)
check_file(args.region_file)
if args.gene_annotations:
check_file(args.gene_annotations)
#INIT
get_name_from_bam=lambda x: os.path.basename(x).replace('.bam','')
if not args.name:
database_id='%s' % get_name_from_bam(args.bam_file)
else:
database_id=args.name
OUTPUT_DIRECTORY='CRISPRessoWGS_on_%s' % database_id
if args.output_folder:
OUTPUT_DIRECTORY=os.path.join(os.path.abspath(args.output_folder),OUTPUT_DIRECTORY)
_jp=lambda filename: os.path.join(OUTPUT_DIRECTORY,filename) #handy function to put a file in the output directory
try:
info('Creating Folder %s' % OUTPUT_DIRECTORY)
os.makedirs(OUTPUT_DIRECTORY)
info('Done!')
except:
warn('Folder %s already exists.' % OUTPUT_DIRECTORY)
log_filename=_jp('CRISPRessoWGS_RUNNING_LOG.txt')
logging.getLogger().addHandler(logging.FileHandler(log_filename))
with open(log_filename,'w+') as outfile:
outfile.write('[Command used]:\n%s\n\n[Execution log]:\n' % ' '.join(sys.argv))
def rreplace(s, old, new):
li = s.rsplit(old)
return new.join(li)
bam_index = ''
#check if bam has the index already
if os.path.exists(args.bam_file.rreplace(".bam",".bai")):
info('Index file for input .bam file exists, skipping generation.')
bam_index = args.bam_file.rreplace(".bam",".bai")
elif os.path.exists(args.bam_file+'.bai'):
info('Index file for input .bam file exists, skipping generation.')
bam_index = args.bam_file+'.bai'
else:
info('Creating index file for input .bam file...')
sb.call('samtools index %s ' % (args.bam_file),shell=True)
bam_index = args.bam_file+'.bai'
#load gene annotation
if args.gene_annotations:
info('Loading gene coordinates from annotation file: %s...' % args.gene_annotations)
try:
df_genes=pd.read_table(args.gene_annotations,compression='gzip')
df_genes.txEnd=df_genes.txEnd.astype(int)
df_genes.txStart=df_genes.txStart.astype(int)
df_genes.head()
except:
info('Failed to load the gene annotations file.')
#Load and validate the REGION FILE
df_regions=pd.read_csv(args.region_file,names=[
'chr_id','bpstart','bpend','Name','sgRNA',
'Expected_HDR','Coding_sequence'],comment='#',sep='\t',dtype={'Name':str})
#remove empty amplicons/lines
df_regions.dropna(subset=['chr_id','bpstart','bpend'],inplace=True)
df_regions.Expected_HDR=df_regions.Expected_HDR.apply(capitalize_sequence)
df_regions.sgRNA=df_regions.sgRNA.apply(capitalize_sequence)
df_regions.Coding_sequence=df_regions.Coding_sequence.apply(capitalize_sequence)
#check or create names
for idx,row in df_regions.iterrows():
if pd.isnull(row.Name):
df_regions.ix[idx,'Name']='_'.join(map(str,[row['chr_id'],row['bpstart'],row['bpend']]))
if not len(df_regions.Name.unique())==df_regions.shape[0]:
raise Exception('The amplicon names should be all distinct!')
df_regions=df_regions.set_index('Name')
#df_regions.index=df_regions.index.str.replace(' ','_')
df_regions.index=df_regions.index.to_series().str.replace(' ','_')
#extract sequence for each region
uncompressed_reference=args.reference_file
if os.path.exists(uncompressed_reference+'.fai'):
info('The index for the reference fasta file is already present! Skipping generation.')
else:
info('Indexing reference file... Please be patient!')
sb.call('samtools faidx %s >>%s 2>&1' % (uncompressed_reference,log_filename),shell=True)
df_regions['sequence']=df_regions.apply(lambda row: get_region_from_fa(row.chr_id,row.bpstart,row.bpend,uncompressed_reference),axis=1)
for idx,row in df_regions.iterrows():
if not pd.isnull(row.sgRNA):
cut_points=[]
for current_guide_seq in row.sgRNA.strip().upper().split(','):
wrong_nt=find_wrong_nt(current_guide_seq)
if wrong_nt:
raise NTException('The sgRNA sequence %s contains wrong characters:%s' % (current_guide_seq, ' '.join(wrong_nt)))
offset_fw=args.quantification_window_center+len(current_guide_seq)-1
offset_rc=(-args.quantification_window_center)-1
cut_points+=[m.start() + offset_fw for \
m in re.finditer(current_guide_seq, row.sequence)]+[m.start() + offset_rc for m in re.finditer(CRISPRessoShared.reverse_complement(current_guide_seq), row.sequence)]
if not cut_points:
df_regions.ix[idx,'sgRNA']=''
df_regions=df_regions.convert_objects(convert_numeric=True)
df_regions.bpstart=df_regions.bpstart.astype(int)
df_regions.bpend=df_regions.bpend.astype(int)
if args.gene_annotations:
df_regions=df_regions.apply(lambda row: find_overlapping_genes(row, df_genes),axis=1)
#extract reads with samtools in that region and create a bam
#create a fasta file with all the trimmed reads
info('\nProcessing each regions...')
ANALYZED_REGIONS=_jp('ANALYZED_REGIONS/')
if not os.path.exists(ANALYZED_REGIONS):
os.mkdir(ANALYZED_REGIONS)
df_regions['n_reads']=0
df_regions['bam_file_with_reads_in_region']=''
df_regions['fastq.gz_file_trimmed_reads_in_region']=''
for idx,row in df_regions.iterrows():
if row['sequence']:
fastq_gz_filename=os.path.join(ANALYZED_REGIONS,'%s.fastq.gz' % clean_filename('REGION_'+str(idx)))
bam_region_filename=os.path.join(ANALYZED_REGIONS,'%s.bam' % clean_filename('REGION_'+str(idx)))
#create place-holder fastq files
open(fastq_gz_filename, 'w+').close()
region='%s:%d-%d' % (row.chr_id,row.bpstart,row.bpend-1)
info('\nExtracting reads in:%s and create the .bam file: %s' % (region,bam_region_filename))
#extract reads in region
cmd=r'''samtools view -b -F 4 %s %s > %s ''' % (args.bam_file, region, bam_region_filename)
#print cmd
sb.call(cmd,shell=True)
#index bam file
cmd=r'''samtools index %s ''' % (bam_region_filename)
#print cmd
sb.call(cmd,shell=True)
info('Trim reads and create a fastq.gz file in: %s' % fastq_gz_filename)
#trim reads in bam and convert in fastq
n_reads=write_trimmed_fastq(bam_region_filename,row['bpstart'],row['bpend'],fastq_gz_filename)
df_regions.ix[idx,'n_reads']=n_reads
df_regions.ix[idx,'bam_file_with_reads_in_region']=bam_region_filename
df_regions.ix[idx,'fastq.gz_file_trimmed_reads_in_region']=fastq_gz_filename
df_regions.fillna('NA').to_csv(_jp('REPORT_READS_ALIGNED_TO_SELECTED_REGIONS_WGS.txt'),sep='\t')
#Run Crispresso
info('\nRunning CRISPResso on each regions...')
crispresso_cmds = []
for idx,row in df_regions.iterrows():
if row['n_reads']>=args.min_reads_to_use_region:
info('\nThe region [%s] has enough reads (%d) mapped to it!' % (idx,row['n_reads']))
crispresso_cmd= args.crispresso_command + ' -r1 %s -a %s -o %s --name %s' %\
(row['fastq.gz_file_trimmed_reads_in_region'],row['sequence'],OUTPUT_DIRECTORY,idx)
if row['sgRNA'] and not pd.isnull(row['sgRNA']):
crispresso_cmd+=' -g %s' % row['sgRNA']
if row['Expected_HDR'] and not pd.isnull(row['Expected_HDR']):
crispresso_cmd+=' -e %s' % row['Expected_HDR']
if row['Coding_sequence'] and not pd.isnull(row['Coding_sequence']):
crispresso_cmd+=' -c %s' % row['Coding_sequence']
crispresso_cmd=CRISPRessoShared.propagate_crispresso_options(crispresso_cmd,crispresso_options_for_wgs,args)
crispresso_cmds.append(crispresso_cmd)
# info('Running CRISPResso:%s' % crispresso_cmd)
# sb.call(crispresso_cmd,shell=True)
else:
info('\nThe region [%s] has too few reads mapped to it (%d)! Skipping the running of CRISPResso!' % (idx,row['n_reads']))
CRISPRessoMultiProcessing.run_crispresso_cmds(crispresso_cmds,args.n_processes,'region',args.skip_failed)
quantification_summary=[]
for idx,row in df_regions.iterrows():
folder_name='CRISPResso_on_%s' % idx
try:
quantification_file,amplicon_names,amplicon_info=CRISPRessoShared.check_output_folder(_jp(folder_name))
for amplicon_name in amplicon_names:
N_TOTAL = float(amplicon_info[amplicon_name]['Total'])
N_UNMODIFIED = float(amplicon_info[amplicon_name]['Unmodified'])
N_MODIFIED = float(amplicon_info[amplicon_name]['Modified'])
quantification_summary.append([idx,amplicon_name,N_UNMODIFIED/N_TOTAL*100,N_MODIFIED/N_TOTAL*100,N_TOTAL,row.n_reads])
except CRISPRessoShared.OutputFolderIncompleteException as e:
quantification_summary.append([idx,"",np.nan,np.nan,np.nan,row.n_reads])
warn('Skipping the folder %s: not enough reads, incomplete, or empty folder.'% folder_name)
df_summary_quantification=pd.DataFrame(quantification_summary,columns=['Name','Amplicon','Unmodified%','Modified%','Reads_aligned','Reads_total'])
df_summary_quantification.fillna('NA').to_csv(_jp('SAMPLES_QUANTIFICATION_SUMMARY.txt'),sep='\t',index=None)
info('All Done!')
print(CRISPRessoShared.get_crispresso_footer())
sys.exit(0)
except Exception as e:
print_stacktrace_if_debug()
error('\n\nERROR: %s' % e)
sys.exit(-1)
def main():
try:
description = [
'~~~CRISPRessoCompare~~~',
'-Comparison of two CRISPResso analyses-'
]
compare_header = r'''
___________________________
| __ __ __ __ __ |
|/ / \|\/||__) /\ |__)|_ |
|\__\__/| || /--\| \ |__ |
|___________________________|
'''
compare_header = CRISPRessoShared.get_crispresso_header(
description, compare_header)
print(compare_header)
parser = argparse.ArgumentParser(
description='CRISPRessoCompare Parameters',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'crispresso_output_folder_1',
type=str,
help='First output folder with CRISPResso analysis')
parser.add_argument(
'crispresso_output_folder_2',
type=str,
help='Second output folder with CRISPResso analysis')
#OPTIONALS
parser.add_argument('-n', '--name', help='Output name', default='')
parser.add_argument('-n1',
'--sample_1_name',
help='Sample 1 name',
default='Sample_1')
parser.add_argument('-n2',
'--sample_2_name',
help='Sample 2 name',
default='Sample_2')
parser.add_argument('-o', '--output_folder', help='', default='')
parser.add_argument(
'--min_frequency_alleles_around_cut_to_plot',
type=float,
help=
'Minimum %% reads required to report an allele in the alleles table plot.',
default=0.2)
parser.add_argument(
'--max_rows_alleles_around_cut_to_plot',
type=int,
help='Maximum number of rows to report in the alleles table plot. ',
default=50)
parser.add_argument(
'--save_also_png',
help='Save also .png images additionally to .pdf files',
action='store_true')
parser.add_argument('--debug',
help='Show debug messages',
action='store_true')
args = parser.parse_args()
debug_flag = args.debug
#check that the CRISPResso output is present and fill amplicon_info
quantification_file_1, amplicon_names_1, amplicon_info_1 = CRISPRessoShared.check_output_folder(
args.crispresso_output_folder_1)
quantification_file_2, amplicon_names_2, amplicon_info_2 = CRISPRessoShared.check_output_folder(
args.crispresso_output_folder_2)
get_name_from_folder = lambda x: os.path.basename(os.path.abspath(
x)).replace('CRISPResso_on_', '')
if not args.name:
database_id = '%s_VS_%s' % (
get_name_from_folder(args.crispresso_output_folder_1),
get_name_from_folder(args.crispresso_output_folder_2))
else:
database_id = args.name
OUTPUT_DIRECTORY = 'CRISPRessoCompare_on_%s' % database_id
if args.output_folder:
OUTPUT_DIRECTORY = os.path.join(
os.path.abspath(args.output_folder), OUTPUT_DIRECTORY)
_jp = lambda filename: os.path.join(
OUTPUT_DIRECTORY, filename
) #handy function to put a file in the output directory
log_filename = _jp('CRISPRessoCompare_RUNNING_LOG.txt')
try:
info('Creating Folder %s' % OUTPUT_DIRECTORY)
os.makedirs(OUTPUT_DIRECTORY)
info('Done!')
except:
warn('Folder %s already exists.' % OUTPUT_DIRECTORY)
log_filename = _jp('CRISPRessoCompare_RUNNING_LOG.txt')
logging.getLogger().addHandler(logging.FileHandler(log_filename))
with open(log_filename, 'w+') as outfile:
outfile.write(
'[Command used]:\nCRISPRessoCompare %s\n\n[Execution log]:\n' %
' '.join(sys.argv))
#LOAD DATA
amplicon_names_in_both = [
amplicon_name for amplicon_name in amplicon_names_1
if amplicon_name in amplicon_names_2
]
n_refs = len(amplicon_names_in_both)
def get_plot_title_with_ref_name(plotTitle, refName):
if n_refs > 1:
return (plotTitle + ": " + refName)
return plotTitle
for amplicon_name in amplicon_names_in_both:
profile_1 = parse_profile(
amplicon_info_1[amplicon_name]['quantification_file'])
profile_2 = parse_profile(
amplicon_info_2[amplicon_name]['quantification_file'])
try:
assert np.all(profile_1[:, 0] == profile_2[:, 0])
except:
raise DifferentAmpliconLengthException(
'Different amplicon lengths for the two amplicons.')
len_amplicon = profile_1.shape[0]
effect_vector_any_1 = profile_1[:, 1]
effect_vector_any_2 = profile_2[:, 1]
cut_points, sgRNA_intervals = load_cut_points_sgRNA_intervals(
args.crispresso_output_folder_1, amplicon_name)
#Quantification comparison barchart
fig = plt.figure(figsize=(30, 15))
n_groups = 2
N_TOTAL_1 = float(amplicon_info_1[amplicon_name]['Total'])
N_UNMODIFIED_1 = float(
amplicon_info_1[amplicon_name]['Unmodified'])
N_MODIFIED_1 = float(amplicon_info_1[amplicon_name]['Modified'])
N_TOTAL_2 = float(amplicon_info_2[amplicon_name]['Total'])
N_UNMODIFIED_2 = float(
amplicon_info_2[amplicon_name]['Unmodified'])
N_MODIFIED_2 = float(amplicon_info_2[amplicon_name]['Modified'])
means_sample_1 = np.array([N_UNMODIFIED_1, N_MODIFIED_1
]) / N_TOTAL_1 * 100
means_sample_2 = np.array([N_UNMODIFIED_2, N_MODIFIED_2
]) / N_TOTAL_2 * 100
ax1 = fig.add_subplot(1, 2, 1)
index = np.arange(n_groups)
bar_width = 0.35
opacity = 0.4
error_config = {'ecolor': '0.3'}
rects1 = ax1.bar(index,
means_sample_1,
bar_width,
alpha=opacity,
color=(0, 0, 1, 0.4),
label=args.sample_1_name)
rects2 = ax1.bar(index + bar_width,
means_sample_2,
bar_width,
alpha=opacity,
color=(1, 0, 0, 0.4),
label=args.sample_2_name)
plt.ylabel('% Sequences')
plt.title(
get_plot_title_with_ref_name(
'%s VS %s' % (args.sample_1_name, args.sample_2_name),
amplicon_name))
plt.xticks(index + bar_width / 2.0, ('Unmodified', 'Modified'))
plt.legend()
# plt.xlim(index[0]-0.2,(index + bar_width)[-1]+bar_width+0.2)
plt.tight_layout()
ax2 = fig.add_subplot(1, 2, 2)
ax2.bar(index,
means_sample_1 - means_sample_2,
bar_width + 0.35,
alpha=opacity,
color=(0, 1, 1, 0.4),
label='')
plt.ylabel('% Sequences Difference')
plt.title(
get_plot_title_with_ref_name(
'%s - %s' % (args.sample_1_name, args.sample_2_name),
amplicon_name))
plt.xticks(index, ['Unmodified', 'Modified'])
# plt.xlim(index[0]-bar_width/2, (index+bar_width)[-1]+2*bar_width)
plt.tight_layout()
plt.savefig(_jp('1.' + amplicon_name +
'.Comparison_Efficiency.pdf'),
bbox_inches='tight')
if args.save_also_png:
plt.savefig(_jp('1.' + amplicon_name +
'.Comparison_Efficiency.png'),
bbox_inches='tight')
#profile comparion
fig = plt.figure(figsize=(20, 10))
ax1 = fig.add_subplot(1, 2, 1)
plt.title(
get_plot_title_with_ref_name('Mutation position distribution',
amplicon_name))
y_max = max(effect_vector_any_1.max(),
effect_vector_any_2.max()) * 1.2
plt.plot(effect_vector_any_1,
color=(0, 0, 1, 0.3),
lw=4,
label='%s combined mutations' % args.sample_1_name)
# plt.hold(True)
plt.plot(effect_vector_any_2,
color=(1, 0, 0, 0.3),
lw=4,
label='%s combined mutations' % args.sample_2_name)
if cut_points:
for idx, cut_point in enumerate(cut_points):
if idx == 0:
plt.plot([cut_point, cut_point], [0, y_max],
'--k',
lw=2,
label='Predicted cleavage position')
else:
plt.plot([cut_point, cut_point], [0, y_max],
'--k',
lw=2,
label='_nolegend_')
for idx, sgRNA_int in enumerate(sgRNA_intervals):
if idx == 0:
plt.plot([sgRNA_int[0], sgRNA_int[1]], [0, 0],
lw=10,
c=(0, 0, 0, 0.15),
label='sgRNA')
else:
plt.plot([sgRNA_int[0], sgRNA_int[1]], [0, 0],
lw=10,
c=(0, 0, 0, 0.15),
label='_nolegend_')
lgd = plt.legend(loc='center',
bbox_to_anchor=(0.5, -0.3),
ncol=1,
fancybox=True,
shadow=False)
plt.xticks(
np.arange(0, len_amplicon,
max(3, (len_amplicon / 6) -
(len_amplicon / 6) % 5)).astype(int))
plt.xlabel('Reference amplicon position (bp)')
plt.ylabel('Sequences %')
plt.ylim(0, max(1, y_max))
plt.xlim(xmax=len_amplicon - 1)
ax2 = fig.add_subplot(1, 2, 2)
effect_vector_any_diff = effect_vector_any_1 - effect_vector_any_2
y_max = effect_vector_any_diff.max() * 1.2
y_min = effect_vector_any_diff.min() * 1.2
plt.title(
get_plot_title_with_ref_name(
'%s - %s' % (args.sample_1_name, args.sample_2_name),
amplicon_name))
plt.plot(effect_vector_any_diff,
color=(0, 1, 0, 0.4),
lw=3,
label='Difference')
if cut_points:
for idx, cut_point in enumerate(cut_points):
if idx == 0:
plt.plot(
[cut_point, cut_point],
[min(-1, y_min), max(1, y_max)],
'--k',
lw=2,
label='Predicted cleavage position')
else:
plt.plot(
[cut_point, cut_point],
[min(-1, y_min), max(1, y_max)],
'--k',
lw=2,
label='_nolegend_')
for idx, sgRNA_int in enumerate(sgRNA_intervals):
if idx == 0:
plt.plot(
[sgRNA_int[0], sgRNA_int[1]],
[min(-1, y_min), min(-1, y_min)],
lw=10,
c=(0, 0, 0, 0.15),
label='sgRNA')
else:
plt.plot(
[sgRNA_int[0], sgRNA_int[1]],
[min(-1, y_min), min(-1, y_min)],
lw=10,
c=(0, 0, 0, 0.15),
label='_nolegend_')
lgd2 = plt.legend(loc='center',
bbox_to_anchor=(0.5, -0.2),
ncol=1,
fancybox=True,
shadow=False)
plt.xticks(
np.arange(0, len_amplicon,
max(3, (len_amplicon / 6) -
(len_amplicon / 6) % 5)).astype(int))
plt.xlabel('Reference amplicon position (bp)')
plt.ylabel('Sequences Difference %')
plt.xlim(xmax=len_amplicon - 1)
plt.ylim(min(-1, y_min), max(1, y_max))
plt.savefig(_jp(
'2.' + amplicon_name +
'.Comparison_Combined_Insertion_Deletion_Substitution_Locations.pdf'
),
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
if args.save_also_png:
plt.savefig(_jp(
'2.' + amplicon_name +
'.Comparison_Insertion_Deletion_Substitution_Locations.png'
),
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
mod_file_1 = amplicon_info_1[amplicon_name][
'modification_count_file']
amp_seq_1, mod_freqs_1 = CRISPRessoShared.parse_count_file(
mod_file_1)
mod_file_2 = amplicon_info_2[amplicon_name][
'modification_count_file']
amp_seq_2, mod_freqs_2 = CRISPRessoShared.parse_count_file(
mod_file_2)
consensus_sequence = amp_seq_1
if amp_seq_2 != consensus_sequence:
raise DifferentAmpliconLengthException(
'Different amplicon lengths for the two amplicons.')
for mod in [
'Insertions', 'Deletions', 'Substitutions',
'All_modifications'
]:
mod_counts_1 = np.array(mod_freqs_1[mod], dtype=float)
tot_counts_1 = np.array(mod_freqs_1['Total'], dtype=float)
unmod_counts_1 = tot_counts_1 - mod_counts_1
mod_counts_2 = np.array(mod_freqs_2[mod], dtype=float)
tot_counts_2 = np.array(mod_freqs_2['Total'], dtype=float)
unmod_counts_2 = tot_counts_2 - mod_counts_2
fisher_results = [
stats.fisher_exact([[z[0], z[1]], [z[2], z[3]]])
if max(z) > 0 else [nan, 1.0]
for z in zip(mod_counts_1, unmod_counts_1, mod_counts_2,
unmod_counts_2)
]
oddsratios, pvalues = [a for a, b in fisher_results
], [b for a, b in fisher_results]
mod_df = []
row = [args.sample_1_name + '_' + mod]
row.extend(mod_counts_1)
mod_df.append(row)
row = [args.sample_1_name + '_total']
row.extend(tot_counts_1)
mod_df.append(row)
row = [args.sample_2_name + '_' + mod]
row.extend(mod_counts_2)
mod_df.append(row)
row = [args.sample_2_name + '_total']
row.extend(tot_counts_2)
mod_df.append(row)
row = ['odds_ratios']
row.extend(oddsratios)
mod_df.append(row)
row = ['pvalues']
row.extend(pvalues)
mod_df.append(row)
colnames = ['Reference']
colnames.extend(list(consensus_sequence))
mod_df = pd.DataFrame(mod_df, columns=colnames)
# mod_df = pd.concat([mod_df.iloc[:,0:2], mod_df.iloc[:,2:].apply(pd.to_numeric)],axis=1)
#write to file
mod_df.to_csv(_jp(amplicon_name + '.' + mod +
'_quantification.txt'),
sep='\t',
index=None)
#plot
fig = plt.figure(figsize=(20, 10))
ax1 = fig.add_subplot(2, 1, 1)
diff = np.divide(mod_counts_1, tot_counts_1) - np.divide(
mod_counts_2, tot_counts_2)
diff_plot = ax1.plot(diff,
color=(0, 1, 0, 0.4),
lw=3,
label='Difference')
ax1.set_title(
get_plot_title_with_ref_name(
'%s: %s - %s' %
(mod, args.sample_1_name, args.sample_2_name),
amplicon_name))
ax1.set_xticks(
np.arange(
0, len_amplicon,
max(3, (len_amplicon / 6) -
(len_amplicon / 6) % 5)).astype(int))
ax1.set_ylabel('Sequences Difference %')
ax1.set_xlim(xmin=0, xmax=len_amplicon - 1)
pvalues = np.array(pvalues)
min_nonzero = np.min(pvalues[np.nonzero(pvalues)])
pvalues[pvalues == 0] = min_nonzero
#ax2 = ax1.twinx()
ax2 = fig.add_subplot(2, 1, 2)
pval_plot = ax2.plot(-1 * np.log10(pvalues),
color=(1, 0, 0, 0.4),
lw=2,
label='-log10 P-value')
ax2.set_ylabel('-log10 P-value')
ax2.set_xlim(xmin=0, xmax=len_amplicon - 1)
ax2.set_xticks(
np.arange(
0, len_amplicon,
max(3, (len_amplicon / 6) -
(len_amplicon / 6) % 5)).astype(int))
ax2.set_xlabel('Reference amplicon position (bp)')
#bonferroni correction
corrected_p = -1 * np.log10(
0.01 / float(len(consensus_sequence)))
cutoff_plot = ax2.plot([0, len(consensus_sequence)],
[corrected_p, corrected_p],
color='k',
dashes=(5, 10),
label='Bonferronni corrected cutoff')
plots = diff_plot + pval_plot + cutoff_plot
diff_y_min, diff_y_max = ax1.get_ylim()
p_y_min, p_y_max = ax2.get_ylim()
if cut_points:
for idx, cut_point in enumerate(cut_points):
if idx == 0:
plot_cleavage = ax1.plot(
[cut_point, cut_point],
[diff_y_min, diff_y_max],
'--k',
lw=2,
label='Predicted cleavage position')
ax2.plot([cut_point, cut_point],
[p_y_min, p_y_max],
'--k',
lw=2,
label='Predicted cleavage position')
plots = plots + plot_cleavage
else:
ax1.plot([cut_point, cut_point],
[diff_y_min, diff_y_max],
'--k',
lw=2,
label='_nolegend_')
ax2.plot([cut_point, cut_point],
[diff_y_min, diff_y_max],
'--k',
lw=2,
label='_nolegend_')
for idx, sgRNA_int in enumerate(sgRNA_intervals):
if idx == 0:
p2 = ax1.plot([sgRNA_int[0], sgRNA_int[1]],
[diff_y_min, diff_y_min],
lw=10,
c=(0, 0, 0, 0.15),
label='sgRNA')
ax2.plot([sgRNA_int[0], sgRNA_int[1]],
[p_y_min, p_y_min],
lw=10,
c=(0, 0, 0, 0.15),
label='sgRNA')
plots = plots + p2
else:
ax1.plot([sgRNA_int[0], sgRNA_int[1]],
[diff_y_min, diff_y_min],
lw=10,
c=(0, 0, 0, 0.15),
label='_nolegend_')
ax2.plot([sgRNA_int[0], sgRNA_int[1]],
[p_y_min, p_y_min],
lw=10,
c=(0, 0, 0, 0.15),
label='_nolegend_')
labs = [p.get_label() for p in plots]
lgd = plt.legend(plots,
labs,
loc='upper center',
bbox_to_anchor=(0.5, -0.2),
ncol=1,
fancybox=True,
shadow=False)
plt.savefig(_jp('2.' + amplicon_name + '.' + mod +
'.quantification.pdf'),
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
if args.save_also_png:
plt.savefig(_jp('2.' + amplicon_name + '.' + mod +
'.quantification.png'),
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
#create merged heatmaps for each cut site
allele_files_1 = amplicon_info_1[amplicon_name]['allele_files']
allele_files_2 = amplicon_info_2[amplicon_name]['allele_files']
for allele_file_1 in allele_files_1:
allele_file_1_name = os.path.split(allele_file_1)[
1] #get file part of path
for allele_file_2 in allele_files_2:
allele_file_2_name = os.path.split(allele_file_2)[
1] #get file part of path
#if files are the same (same amplicon, cut site, guide), run comparison
if allele_file_1_name == allele_file_2_name:
df1 = pd.read_csv(allele_file_1, sep="\t")
df2 = pd.read_csv(allele_file_2, sep="\t")
#find unmodified reference for comparison (if it exists)
ref_seq_around_cut = ""
if len(df1.loc[df1['Reference_Sequence'].str.contains(
'-') == False]) > 0:
ref_seq_around_cut = df1.loc[
df1['Reference_Sequence'].str.contains('-') ==
False]['Reference_Sequence'].iloc[0]
#otherwise figure out which sgRNA was used for this comparison
elif len(df2.loc[df2['Reference_Sequence'].str.
contains('-') == False]) > 0:
ref_seq_around_cut = df2.loc[
df2['Reference_Sequence'].str.contains('-') ==
False]['Reference_Sequence'].iloc[0]
else:
seq_len = df2[df2['Unedited'] ==
True]['Reference_Sequence'].iloc[0]
for sgRNA_interval, cut_point in zip(
sgRNA_intervals, cut_points):
sgRNA_seq = consensus_sequence[
sgRNA_interval[0]:sgRNA_interval[1]]
if sgRNA_seq in allele_file_1_name:
this_sgRNA_seq = sgRNA_seq
this_cut_point = cut_point
ref_seq_around_cut = consensus_sequence[max(
0, this_cut_point -
args.offset_around_cut_to_plot +
1):min(
len(reference_seq), cut_point +
args.offset_around_cut_to_plot +
1)]
break
merged = pd.merge(df1,
df2,
on=[
'Aligned_Sequence',
'Reference_Sequence', 'Unedited',
'n_deleted', 'n_inserted',
'n_mutated'
],
suffixes=('_' + args.sample_1_name,
'_' + args.sample_2_name),
how='outer')
quant_cols = [
'#Reads_' + args.sample_1_name,
'%Reads_' + args.sample_1_name,
'#Reads_' + args.sample_2_name,
'%Reads_' + args.sample_2_name
]
merged[quant_cols] = merged[quant_cols].fillna(0)
lfc_error = 0.1
merged['each_LFC'] = np.log2(
((merged['%Reads_' + args.sample_1_name] +
lfc_error) /
(merged['%Reads_' + args.sample_2_name] +
lfc_error)).astype(float)).replace(
[np.inf, np.NaN], 0)
merged = merged.reset_index().set_index(
'Aligned_Sequence')
output_root = allele_file_1_name.replace(".txt", "")
merged.to_csv(_jp(output_root + ".txt"),
sep="\t",
index=None)
CRISPRessoPlot.plot_alleles_table_compare(
ref_seq_around_cut,
merged.sort_values(['each_LFC'], ascending=True),
args.sample_1_name,
args.sample_2_name,
_jp('3.' + output_root + "_top"),
MIN_FREQUENCY=args.
min_frequency_alleles_around_cut_to_plot,
MAX_N_ROWS=args.
max_rows_alleles_around_cut_to_plot,
SAVE_ALSO_PNG=args.save_also_png)
CRISPRessoPlot.plot_alleles_table_compare(
ref_seq_around_cut,
merged.sort_values(['each_LFC'], ascending=False),
args.sample_1_name,
args.sample_2_name,
_jp('3.' + output_root + "_bottom"),
MIN_FREQUENCY=args.
min_frequency_alleles_around_cut_to_plot,
MAX_N_ROWS=args.
max_rows_alleles_around_cut_to_plot,
SAVE_ALSO_PNG=args.save_also_png)
info('All Done!')
print(CRISPRessoShared.get_crispresso_footer())
sys.exit(0)
except Exception as e:
debug_flag = False
if 'args' in vars() and 'debug' in args:
debug_flag = args.debug
if debug_flag:
traceback.print_exc(file=sys.stdout)
error('\n\nERROR: %s' % e)
sys.exit(-1)
def main():
try:
description = ['~~~CRISPRessoPooled~~~','-Analysis of CRISPR/Cas9 outcomes from POOLED deep sequencing data-']
pooled_string = r'''
_______________________
| __ __ __ __ __ |
||__)/ \/ \| |_ | \ |
|| \__/\__/|__|__|__/ |
|_______________________|
'''
print(CRISPRessoShared.get_crispresso_header(description,pooled_string))
parser = CRISPRessoShared.getCRISPRessoArgParser(parserTitle = 'CRISPRessoPooled Parameters',requiredParams={'fastq_r1':True})
parser.add_argument('-f','--amplicons_file', type=str, help='Amplicons description file. This file is a tab-delimited text file with up to 5 columns (2 required):\
\nAMPLICON_NAME: an identifier for the amplicon (must be unique)\nAMPLICON_SEQUENCE: amplicon sequence used in the experiment\n\
\nsgRNA_SEQUENCE (OPTIONAL): sgRNA sequence used for this amplicon without the PAM sequence. Multiple guides can be given separated by commas and not spaces. If not available enter NA.\
\nEXPECTED_AMPLICON_AFTER_HDR (OPTIONAL): expected amplicon sequence in case of HDR. If not available enter NA.\
\nCODING_SEQUENCE (OPTIONAL): Subsequence(s) of the amplicon corresponding to coding sequences. If more than one separate them by commas and not spaces. If not available enter NA.', default='')
#tool specific optional
parser.add_argument('--gene_annotations', type=str, help='Gene Annotation Table from UCSC Genome Browser Tables (http://genome.ucsc.edu/cgi-bin/hgTables?command=start), \
please select as table "knownGene", as output format "all fields from selected table" and as file returned "gzip compressed"', default='')
parser.add_argument('-p','--n_processes',type=int, help='Specify the number of processes to use for Bowtie2.\
Please use with caution since increasing this parameter will increase significantly the memory required to run CRISPResso.',default=1)
parser.add_argument('-x','--bowtie2_index', type=str, help='Basename of Bowtie2 index for the reference genome', default='')
parser.add_argument('--bowtie2_options_string', type=str, help='Override options for the Bowtie2 alignment command',default=' -k 1 --end-to-end -N 0 --np 0 ')
parser.add_argument('--min_reads_to_use_region', type=float, help='Minimum number of reads that align to a region to perform the CRISPResso analysis', default=1000)
parser.add_argument('--skip_failed', help='Continue with pooled analysis even if one sample fails',action='store_true')
parser.add_argument('--crispresso_command', help='CRISPResso command to call',default='CRISPResso')
args = parser.parse_args()
crispresso_options = CRISPRessoShared.get_crispresso_options()
options_to_ignore = set(['fastq_r1','fastq_r2','amplicon_seq','amplicon_name','output_folder','name'])
crispresso_options_for_pooled = list(crispresso_options-options_to_ignore)
info('Checking dependencies...')
if check_samtools() and check_bowtie2():
info('\n All the required dependencies are present!')
else:
sys.exit(1)
#check files
check_file(args.fastq_r1)
if args.fastq_r2:
check_file(args.fastq_r2)
if args.bowtie2_index:
check_file(args.bowtie2_index+'.1.bt2')
if args.amplicons_file:
check_file(args.amplicons_file)
if args.gene_annotations:
check_file(args.gene_annotations)
if args.amplicons_file and not args.bowtie2_index:
RUNNING_MODE='ONLY_AMPLICONS'
info('Only the Amplicon description file was provided. The analysis will be perfomed using only the provided amplicons sequences.')
elif args.bowtie2_index and not args.amplicons_file:
RUNNING_MODE='ONLY_GENOME'
info('Only the bowtie2 reference genome index file was provided. The analysis will be perfomed using only genomic regions where enough reads align.')
elif args.bowtie2_index and args.amplicons_file:
RUNNING_MODE='AMPLICONS_AND_GENOME'
info('Amplicon description file and bowtie2 reference genome index files provided. The analysis will be perfomed using the reads that are aligned ony to the amplicons provided and not to other genomic regions.')
else:
error('Please provide the amplicons description file (-f or --amplicons_file option) or the bowtie2 reference genome index file (-x or --bowtie2_index option) or both.')
sys.exit(1)
####TRIMMING AND MERGING
get_name_from_fasta=lambda x: os.path.basename(x).replace('.fastq','').replace('.gz','')
if not args.name:
if args.fastq_r2!='':
database_id='%s_%s' % (get_name_from_fasta(args.fastq_r1),get_name_from_fasta(args.fastq_r2))
else:
database_id='%s' % get_name_from_fasta(args.fastq_r1)
else:
database_id=args.name
OUTPUT_DIRECTORY='CRISPRessoPooled_on_%s' % database_id
if args.output_folder:
OUTPUT_DIRECTORY=os.path.join(os.path.abspath(args.output_folder),OUTPUT_DIRECTORY)
_jp=lambda filename: os.path.join(OUTPUT_DIRECTORY,filename) #handy function to put a file in the output directory
try:
info('Creating Folder %s' % OUTPUT_DIRECTORY)
os.makedirs(OUTPUT_DIRECTORY)
info('Done!')
except:
warn('Folder %s already exists.' % OUTPUT_DIRECTORY)
log_filename=_jp('CRISPRessoPooled_RUNNING_LOG.txt')
logging.getLogger().addHandler(logging.FileHandler(log_filename))
with open(log_filename,'w+') as outfile:
outfile.write('[Command used]:\nCRISPRessoPooled %s\n\n[Execution log]:\n' % ' '.join(sys.argv))
if args.fastq_r2=='': #single end reads
#check if we need to trim
if not args.trim_sequences:
#create a symbolic link
symlink_filename=_jp(os.path.basename(args.fastq_r1))
force_symlink(os.path.abspath(args.fastq_r1),symlink_filename)
output_forward_filename=symlink_filename
else:
output_forward_filename=_jp('reads.trimmed.fq.gz')
#Trimming with trimmomatic
cmd='java -jar %s SE -phred33 %s %s %s >>%s 2>&1'\
% (get_data('trimmomatic-0.33.jar'),args.fastq_r1,
output_forward_filename,
args.trimmomatic_options_string.replace('NexteraPE-PE.fa','TruSeq3-SE.fa'),
log_filename)
#print cmd
TRIMMOMATIC_STATUS=sb.call(cmd,shell=True)
if TRIMMOMATIC_STATUS:
raise TrimmomaticException('TRIMMOMATIC failed to run, please check the log file.')
processed_output_filename=output_forward_filename
else:#paired end reads case
if not args.trim_sequences:
output_forward_paired_filename=args.fastq_r1
output_reverse_paired_filename=args.fastq_r2
else:
info('Trimming sequences with Trimmomatic...')
output_forward_paired_filename=_jp('output_forward_paired.fq.gz')
output_forward_unpaired_filename=_jp('output_forward_unpaired.fq.gz')
output_reverse_paired_filename=_jp('output_reverse_paired.fq.gz')
output_reverse_unpaired_filename=_jp('output_reverse_unpaired.fq.gz')
#Trimming with trimmomatic
cmd='java -jar %s PE -phred33 %s %s %s %s %s %s %s >>%s 2>&1'\
% (get_data('trimmomatic-0.33.jar'),
args.fastq_r1,args.fastq_r2,output_forward_paired_filename,
output_forward_unpaired_filename,output_reverse_paired_filename,
output_reverse_unpaired_filename,args.trimmomatic_options_string,log_filename)
#print cmd
TRIMMOMATIC_STATUS=sb.call(cmd,shell=True)
if TRIMMOMATIC_STATUS:
raise TrimmomaticException('TRIMMOMATIC failed to run, please check the log file.')
info('Done!')
max_overlap_string = ""
min_overlap_string = ""
if args.max_paired_end_reads_overlap:
max_overlap_string = "--max-overlap " + str(args.max_paired_end_reads_overlap)
if args.min_paired_end_reads_overlap:
min_overlap_string = args.min_paired_end_reads_overlap
#Merging with Flash
info('Merging paired sequences with Flash...')
cmd='flash %s %s %s %s -z -d %s >>%s 2>&1' %\
(output_forward_paired_filename,
output_reverse_paired_filename,
max_overlap_string,
max_overlap_string,
OUTPUT_DIRECTORY,log_filename)
FLASH_STATUS=sb.call(cmd,shell=True)
if FLASH_STATUS:
raise FlashException('Flash failed to run, please check the log file.')
info('Done!')
flash_hist_filename=_jp('out.hist')
flash_histogram_filename=_jp('out.histogram')
flash_not_combined_1_filename=_jp('out.notCombined_1.fastq.gz')
flash_not_combined_2_filename=_jp('out.notCombined_2.fastq.gz')
processed_output_filename=_jp('out.extendedFrags.fastq.gz')
#count reads
N_READS_INPUT=get_n_reads_fastq(args.fastq_r1)
N_READS_AFTER_PREPROCESSING=get_n_reads_fastq(processed_output_filename)
#load gene annotation
if args.gene_annotations:
info('Loading gene coordinates from annotation file: %s...' % args.gene_annotations)
try:
df_genes=pd.read_table(args.gene_annotations,compression='gzip')
df_genes.txEnd=df_genes.txEnd.astype(int)
df_genes.txStart=df_genes.txStart.astype(int)
df_genes.head()
except:
info('Failed to load the gene annotations file.')
if RUNNING_MODE=='ONLY_AMPLICONS' or RUNNING_MODE=='AMPLICONS_AND_GENOME':
#load and validate template file
df_template=pd.read_csv(args.amplicons_file,names=[
'Name','Amplicon_Sequence','sgRNA',
'Expected_HDR','Coding_sequence'],comment='#',sep='\t',dtype={'Name':str})
if df_template.iloc[0,1].lower() == "amplicon_sequence":
df_template.drop(0,axis=0,inplace=True)
info('Detected header in amplicon file.')
#remove empty amplicons/lines
df_template.dropna(subset=['Amplicon_Sequence'],inplace=True)
df_template.dropna(subset=['Name'],inplace=True)
df_template.Amplicon_Sequence=df_template.Amplicon_Sequence.apply(capitalize_sequence)
df_template.Expected_HDR=df_template.Expected_HDR.apply(capitalize_sequence)
df_template.sgRNA=df_template.sgRNA.apply(capitalize_sequence)
df_template.Coding_sequence=df_template.Coding_sequence.apply(capitalize_sequence)
if not len(df_template.Amplicon_Sequence.unique())==df_template.shape[0]:
raise Exception('The amplicons should be all distinct!')
if not len(df_template.Name.unique())==df_template.shape[0]:
raise Exception('The amplicon names should be all distinct!')
df_template=df_template.set_index('Name')
df_template.index=df_template.index.to_series().str.replace(' ','_')
for idx,row in df_template.iterrows():
wrong_nt=find_wrong_nt(row.Amplicon_Sequence)
if wrong_nt:
raise NTException('The amplicon sequence %s contains wrong characters:%s' % (idx,' '.join(wrong_nt)))
if not pd.isnull(row.sgRNA):
cut_points=[]
for current_guide_seq in row.sgRNA.strip().upper().split(','):
wrong_nt=find_wrong_nt(current_guide_seq)
if wrong_nt:
raise NTException('The sgRNA sequence %s contains wrong characters:%s' % (current_guide_seq, ' '.join(wrong_nt)))
offset_fw=args.quantification_window_center+len(current_guide_seq)-1
offset_rc=(-args.quantification_window_center)-1
cut_points+=[m.start() + offset_fw for \
m in re.finditer(current_guide_seq, row.Amplicon_Sequence)]+[m.start() + offset_rc for m in re.finditer(reverse_complement(current_guide_seq), row.Amplicon_Sequence)]
if not cut_points:
warn('\nThe guide sequence/s provided: %s is(are) not present in the amplicon sequence:%s! \nNOTE: The guide will be ignored for the analysis. Please check your input!' % (row.sgRNA,row.Amplicon_Sequence))
df_template.ix[idx,'sgRNA']=''
if RUNNING_MODE=='ONLY_AMPLICONS':
#create a fasta file with all the amplicons
amplicon_fa_filename=_jp('AMPLICONS.fa')
fastq_gz_amplicon_filenames=[]
with open(amplicon_fa_filename,'w+') as outfile:
for idx,row in df_template.iterrows():
if row['Amplicon_Sequence']:
outfile.write('>%s\n%s\n' %(clean_filename('AMPL_'+idx),row['Amplicon_Sequence']))
#create place-holder fastq files
fastq_gz_amplicon_filenames.append(_jp('%s.fastq.gz' % clean_filename('AMPL_'+idx)))
open(fastq_gz_amplicon_filenames[-1], 'w+').close()
df_template['Demultiplexed_fastq.gz_filename']=fastq_gz_amplicon_filenames
info('Creating a custom index file with all the amplicons...')
custom_index_filename=_jp('CUSTOM_BOWTIE2_INDEX')
sb.call('bowtie2-build %s %s >>%s 2>&1' %(amplicon_fa_filename,custom_index_filename,log_filename), shell=True)
#align the file to the amplicons (MODE 1)
info('Align reads to the amplicons...')
bam_filename_amplicons= _jp('CRISPResso_AMPLICONS_ALIGNED.bam')
aligner_command= 'bowtie2 -x %s -p %s %s -U %s 2>>%s | samtools view -bS - > %s' %(custom_index_filename,args.n_processes,args.bowtie2_options_string,processed_output_filename,log_filename,bam_filename_amplicons)
info('Alignment command: ' + aligner_command)
sb.call(aligner_command,shell=True)
N_READS_ALIGNED=get_n_aligned_bam(bam_filename_amplicons)
s1=r"samtools view -F 4 %s 2>>%s | grep -v ^'@'" % (bam_filename_amplicons,log_filename)
s2=r'''|awk '{ gzip_filename=sprintf("gzip >> OUTPUTPATH%s.fastq.gz",$3);\
print "@"$1"\n"$10"\n+\n"$11 | gzip_filename;}' '''
cmd=s1+s2.replace('OUTPUTPATH',_jp(''))
sb.call(cmd,shell=True)
info('Demultiplex reads and run CRISPResso on each amplicon...')
n_reads_aligned_amplicons=[]
crispresso_cmds = []
for idx,row in df_template.iterrows():
info('\n Processing:%s' %idx)
n_reads_aligned_amplicons.append(get_n_reads_fastq(row['Demultiplexed_fastq.gz_filename']))
crispresso_cmd= args.crispresso_command + ' -r1 %s -a %s -o %s --name %s' % (row['Demultiplexed_fastq.gz_filename'],row['Amplicon_Sequence'],OUTPUT_DIRECTORY,idx)
if n_reads_aligned_amplicons[-1]>args.min_reads_to_use_region:
if row['sgRNA'] and not pd.isnull(row['sgRNA']):
crispresso_cmd+=' -g %s' % row['sgRNA']
if row['Expected_HDR'] and not pd.isnull(row['Expected_HDR']):
crispresso_cmd+=' -e %s' % row['Expected_HDR']
if row['Coding_sequence'] and not pd.isnull(row['Coding_sequence']):
crispresso_cmd+=' -c %s' % row['Coding_sequence']
crispresso_cmd=CRISPRessoShared.propagate_crispresso_options(crispresso_cmd,crispresso_options_for_pooled,args)
crispresso_cmds.append(crispresso_cmd)
else:
warn('Skipping amplicon [%s] because no reads align to it\n'% idx)
CRISPRessoMultiProcessing.run_crispresso_cmds(crispresso_cmds,args.n_processes,'amplicon',args.skip_failed)
df_template['n_reads']=n_reads_aligned_amplicons
df_template['n_reads_aligned_%']=df_template['n_reads']/float(N_READS_ALIGNED)*100
df_template.fillna('NA').to_csv(_jp('REPORT_READS_ALIGNED_TO_AMPLICONS.txt'),sep='\t')
if RUNNING_MODE=='AMPLICONS_AND_GENOME':
print 'Mapping amplicons to the reference genome...'
#find the locations of the amplicons on the genome and their strand and check if there are mutations in the reference genome
additional_columns=[]
for idx,row in df_template.iterrows():
fields_to_append=list(np.take(get_align_sequence(row.Amplicon_Sequence, args.bowtie2_index).split('\t'),[0,1,2,3,5]))
if fields_to_append[0]=='*':
info('The amplicon [%s] is not mappable to the reference genome provided!' % idx )
additional_columns.append([idx,'NOT_ALIGNED',0,-1,'+',''])
else:
additional_columns.append([idx]+fields_to_append)
info('The amplicon [%s] was mapped to: %s ' % (idx,' '.join(fields_to_append[:3]) ))
df_template=df_template.join(pd.DataFrame(additional_columns,columns=['Name','chr_id','bpstart','bpend','strand','Reference_Sequence']).set_index('Name'))
df_template.bpstart=df_template.bpstart.astype(int)
df_template.bpend=df_template.bpend.astype(int)
#Check reference is the same otherwise throw a warning
for idx,row in df_template.iterrows():
if row.Amplicon_Sequence != row.Reference_Sequence and row.Amplicon_Sequence != reverse_complement(row.Reference_Sequence):
warn('The amplicon sequence %s provided:\n%s\n\nis different from the reference sequence(both strand):\n\n%s\n\n%s\n' %(row.name,row.Amplicon_Sequence,row.Amplicon_Sequence,reverse_complement(row.Amplicon_Sequence)))
if RUNNING_MODE=='ONLY_GENOME' or RUNNING_MODE=='AMPLICONS_AND_GENOME':
###HERE we recreate the uncompressed genome file if not available###
#check you have all the files for the genome and create a fa idx for samtools
uncompressed_reference=args.bowtie2_index+'.fa'
#if not os.path.exists(GENOME_LOCAL_FOLDER):
# os.mkdir(GENOME_LOCAL_FOLDER)
if os.path.exists(uncompressed_reference):
info('The uncompressed reference fasta file for %s is already present! Skipping generation.' % args.bowtie2_index)
else:
#uncompressed_reference=os.path.join(GENOME_LOCAL_FOLDER,'UNCOMPRESSED_REFERENCE_FROM_'+args.bowtie2_index.replace('/','_')+'.fa')
info('Extracting uncompressed reference from the provided bowtie2 index since it is not available... Please be patient!')
cmd_to_uncompress='bowtie2-inspect %s > %s 2>>%s' % (args.bowtie2_index,uncompressed_reference,log_filename)
sb.call(cmd_to_uncompress,shell=True)
info('Indexing fasta file with samtools...')
#!samtools faidx {uncompressed_reference}
sb.call('samtools faidx %s 2>>%s ' % (uncompressed_reference,log_filename),shell=True)
#####CORRECT ONE####
#align in unbiased way the reads to the genome
if RUNNING_MODE=='ONLY_GENOME' or RUNNING_MODE=='AMPLICONS_AND_GENOME':
info('Aligning reads to the provided genome index...')
bam_filename_genome = _jp('%s_GENOME_ALIGNED.bam' % database_id)
aligner_command= 'bowtie2 -x %s -p %s %s -U %s 2>>%s| samtools view -bS - > %s' %(args.bowtie2_index,args.n_processes,args.bowtie2_options_string,processed_output_filename,log_filename,bam_filename_genome)
info('aligning with command: ' + aligner_command)
sb.call(aligner_command,shell=True)
N_READS_ALIGNED=get_n_aligned_bam(bam_filename_genome)
#REDISCOVER LOCATIONS and DEMULTIPLEX READS
MAPPED_REGIONS=_jp('MAPPED_REGIONS/')
if not os.path.exists(MAPPED_REGIONS):
os.mkdir(MAPPED_REGIONS)
s1=r'''samtools view -F 0x0004 %s 2>>%s |''' % (bam_filename_genome,log_filename)+\
r'''awk '{OFS="\t"; bpstart=$4; bpend=bpstart; split ($6,a,"[MIDNSHP]"); n=0;\
for (i=1; i in a; i++){\
n+=1+length(a[i]);\
if (substr($6,n,1)=="S"){\
if (bpend==$4)\
bpstart-=a[i];\
else
bpend+=a[i];
}\
else if( (substr($6,n,1)!="I") && (substr($6,n,1)!="H") )\
bpend+=a[i];\
}\
if ( ($2 % 32)>=16)\
print $3,bpstart,bpend,"-",$1,$10,$11;\
else\
print $3,bpstart,bpend,"+",$1,$10,$11;}' | '''
s2=r''' sort -k1,1 -k2,2n | awk \
'BEGIN{chr_id="NA";bpstart=-1;bpend=-1; fastq_filename="NA"}\
{ if ( (chr_id!=$1) || (bpstart!=$2) || (bpend!=$3) )\
{\
if (fastq_filename!="NA") {close(fastq_filename); system("gzip -f "fastq_filename)}\
chr_id=$1; bpstart=$2; bpend=$3;\
fastq_filename=sprintf("__OUTPUTPATH__REGION_%s_%s_%s.fastq",$1,$2,$3);\
}\
print "@"$5"\n"$6"\n+\n"$7 >> fastq_filename;\
}' '''
cmd=s1+s2.replace('__OUTPUTPATH__',MAPPED_REGIONS)
info('Demultiplexing reads by location...')
sb.call(cmd,shell=True)
#gzip the missing ones
sb.call('gzip -f %s/*.fastq' % MAPPED_REGIONS,shell=True)
'''
The most common use case, where many different target sites are pooled into a single
high-throughput sequencing library for quantification, is not directly addressed by this implementation.
Potential users of CRISPResso would need to write their own code to generate separate input files for processing.
Importantly, this preprocessing code would need to remove any PCR amplification artifacts
(such as amplification of sequences from a gene and a highly similar pseudogene )
which may confound the interpretation of results.
This can be done by mapping of input sequences to a reference genome and removing
those that do not map to the expected genomic location, but is non-trivial for an end-user to implement.
'''
if RUNNING_MODE=='AMPLICONS_AND_GENOME':
files_to_match=glob.glob(os.path.join(MAPPED_REGIONS,'REGION*'))
n_reads_aligned_genome=[]
fastq_region_filenames=[]
crispresso_cmds = []
for idx,row in df_template.iterrows():
info('Processing amplicon: %s' % idx )
#check if we have reads
fastq_filename_region=os.path.join(MAPPED_REGIONS,'REGION_%s_%s_%s.fastq.gz' % (row['chr_id'],row['bpstart'],row['bpend']))
if os.path.exists(fastq_filename_region):
N_READS=get_n_reads_fastq(fastq_filename_region)
n_reads_aligned_genome.append(N_READS)
fastq_region_filenames.append(fastq_filename_region)
files_to_match.remove(fastq_filename_region)
if N_READS>=args.min_reads_to_use_region:
info('\nThe amplicon [%s] has enough reads (%d) mapped to it! Running CRISPResso!\n' % (idx,N_READS))
crispresso_cmd= args.crispresso_command + ' -r1 %s -a %s -o %s --name %s' % (fastq_filename_region,row['Amplicon_Sequence'],OUTPUT_DIRECTORY,idx)
if row['sgRNA'] and not pd.isnull(row['sgRNA']):
crispresso_cmd+=' -g %s' % row['sgRNA']
if row['Expected_HDR'] and not pd.isnull(row['Expected_HDR']):
crispresso_cmd+=' -e %s' % row['Expected_HDR']
if row['Coding_sequence'] and not pd.isnull(row['Coding_sequence']):
crispresso_cmd+=' -c %s' % row['Coding_sequence']
crispresso_cmd=CRISPRessoShared.propagate_crispresso_options(crispresso_cmd,crispresso_options_for_pooled,args)
info('Running CRISPResso:%s' % crispresso_cmd)
crispresso_cmds.append(crispresso_cmd)
else:
warn('The amplicon [%s] has not enough reads (%d) mapped to it! Skipping the execution of CRISPResso!' % (idx,N_READS))
else:
fastq_region_filenames.append('')
n_reads_aligned_genome.append(0)
warn("The amplicon %s doesn't have any read mapped to it!\n Please check your amplicon sequence." % idx)
CRISPRessoMultiProcessing.run_crispresso_cmds(crispresso_cmds,args.n_processes,'amplicon',args.skip_failed)
df_template['Amplicon_Specific_fastq.gz_filename']=fastq_region_filenames
df_template['n_reads']=n_reads_aligned_genome
df_template['n_reads_aligned_%']=df_template['n_reads']/float(N_READS_ALIGNED)*100
if args.gene_annotations:
df_template=df_template.apply(lambda row: find_overlapping_genes(row, df_genes),axis=1)
df_template.fillna('NA').to_csv(_jp('REPORT_READS_ALIGNED_TO_GENOME_AND_AMPLICONS.txt'),sep='\t')
#write another file with the not amplicon regions
info('Reporting problematic regions...')
coordinates=[]
for region in files_to_match:
coordinates.append(os.path.basename(region).replace('.fastq.gz','').replace('.fastq','').split('_')[1:4]+[region,get_n_reads_fastq(region)])
df_regions=pd.DataFrame(coordinates,columns=['chr_id','bpstart','bpend','fastq_file','n_reads'])
df_regions.dropna(inplace=True) #remove regions in chrUn
df_regions['bpstart'] = pd.to_numeric(df_regions['bpstart'])
df_regions['bpend'] = pd.to_numeric(df_regions['bpend'])
df_regions['n_reads'] = pd.to_numeric(df_regions['n_reads'])
df_regions.bpstart=df_regions.bpstart.astype(int)
df_regions.bpend=df_regions.bpend.astype(int)
df_regions['n_reads_aligned_%']=df_regions['n_reads']/float(N_READS_ALIGNED)*100
df_regions['Reference_sequence']=df_regions.apply(lambda row: get_region_from_fa(row.chr_id,row.bpstart,row.bpend,uncompressed_reference),axis=1)
if args.gene_annotations:
info('Checking overlapping genes...')
df_regions=df_regions.apply(lambda row: find_overlapping_genes(row, df_genes),axis=1)
if np.sum(np.array(map(int,pd.__version__.split('.')))*(100,10,1))< 170:
df_regions.sort('n_reads',ascending=False,inplace=True)
else:
df_regions.sort_values(by='n_reads',ascending=False,inplace=True)
df_regions.fillna('NA').to_csv(_jp('REPORTS_READS_ALIGNED_TO_GENOME_NOT_MATCHING_AMPLICONS.txt'),sep='\t',index=None)
if RUNNING_MODE=='ONLY_GENOME' :
#Load regions and build REFERENCE TABLES
info('Parsing the demultiplexed files and extracting locations and reference sequences...')
coordinates=[]
for region in glob.glob(os.path.join(MAPPED_REGIONS,'REGION*.fastq.gz')):
coord_from_filename = os.path.basename(region).replace('.fastq.gz','').split('_')[1:4]
print('ccord from filename: ' + str(coord_from_filename))
if not (coord_from_filename[1].isdigit() and coord_from_filename[2].isdigit()):
warn('Skipping region [%s] because the region name cannot be parsed\n'% region)
continue
coordinates.append(coord_from_filename+[region,get_n_reads_fastq(region)])
df_regions=pd.DataFrame(coordinates,columns=['chr_id','bpstart','bpend','fastq_file','n_reads'])
df_regions.dropna(inplace=True) #remove regions in chrUn
df_regions['bpstart'] = pd.to_numeric(df_regions['bpstart'])
df_regions['bpend'] = pd.to_numeric(df_regions['bpend'])
df_regions['n_reads'] = pd.to_numeric(df_regions['n_reads'])
df_regions.bpstart=df_regions.bpstart.astype(int)
df_regions.bpend=df_regions.bpend.astype(int)
df_regions['sequence']=df_regions.apply(lambda row: get_region_from_fa(row.chr_id,row.bpstart,row.bpend,uncompressed_reference),axis=1)
df_regions['n_reads_aligned_%']=df_regions['n_reads']/float(N_READS_ALIGNED)*100
if args.gene_annotations:
info('Checking overlapping genes...')
df_regions=df_regions.apply(lambda row: find_overlapping_genes(row, df_genes),axis=1)
if np.sum(np.array(map(int,pd.__version__.split('.')))*(100,10,1))< 170:
df_regions.sort('n_reads',ascending=False,inplace=True)
else:
df_regions.sort_values(by='n_reads',ascending=False,inplace=True)
df_regions.fillna('NA').to_csv(_jp('REPORT_READS_ALIGNED_TO_GENOME_ONLY.txt'),sep='\t',index=None)
#run CRISPResso
#demultiplex reads in the amplicons and call crispresso!
info('Running CRISPResso on the regions discovered...')
crispresso_cmds = []
for idx,row in df_regions.iterrows():
if row.n_reads > args.min_reads_to_use_region:
info('\nRunning CRISPResso on: %s-%d-%d...'%(row.chr_id,row.bpstart,row.bpend ))
crispresso_cmd= args.crispresso_command + ' -r1 %s -a %s -o %s' %(row.fastq_file,row.sequence,OUTPUT_DIRECTORY)
crispresso_cmd=CRISPRessoShared.propagate_crispresso_options(crispresso_cmd,crispresso_options_for_pooled,args)
crispresso_cmds.append(crispresso_cmd)
else:
info('Skipping region: %s-%d-%d , not enough reads (%d)' %(row.chr_id,row.bpstart,row.bpend, row.n_reads))
CRISPRessoMultiProcessing.run_crispresso_cmds(crispresso_cmds,args.n_processes,'region',args.skip_failed)
#write alignment statistics
with open(_jp('MAPPING_STATISTICS.txt'),'w+') as outfile:
outfile.write('READS IN INPUTS:%d\nREADS AFTER PREPROCESSING:%d\nREADS ALIGNED:%d' % (N_READS_INPUT,N_READS_AFTER_PREPROCESSING,N_READS_ALIGNED))
quantification_summary=[]
if RUNNING_MODE=='ONLY_AMPLICONS' or RUNNING_MODE=='AMPLICONS_AND_GENOME':
df_final_data=df_template
else:
df_final_data=df_regions
for idx,row in df_final_data.iterrows():
run_name = idx
if RUNNING_MODE=='ONLY_AMPLICONS' or RUNNING_MODE=='AMPLICONS_AND_GENOME':
run_name=idx
else:
run_name='REGION_%s_%d_%d' %(row.chr_id,row.bpstart,row.bpend )
folder_name = 'CRISPResso_on_%s'%run_name
try:
quantification_file,amplicon_names,amplicon_info=CRISPRessoShared.check_output_folder(_jp(folder_name))
for amplicon_name in amplicon_names:
N_TOTAL = float(amplicon_info[amplicon_name]['Total'])
N_UNMODIFIED = float(amplicon_info[amplicon_name]['Unmodified'])
N_MODIFIED = float(amplicon_info[amplicon_name]['Modified'])
quantification_summary.append([run_name,amplicon_name,N_UNMODIFIED/N_TOTAL*100,N_MODIFIED/N_TOTAL*100,N_TOTAL,row.n_reads])
except CRISPRessoShared.OutputFolderIncompleteException as e:
quantification_summary.append([run_name,"",np.nan,np.nan,np.nan,row.n_reads])
warn('Skipping the folder %s: not enough reads, incomplete, or empty folder.'% folder_name)
df_summary_quantification=pd.DataFrame(quantification_summary,columns=['Name','Amplicon','Unmodified%','Modified%','Reads_aligned','Reads_total'])
df_summary_quantification.fillna('NA').to_csv(_jp('SAMPLES_QUANTIFICATION_SUMMARY.txt'),sep='\t',index=None)
if RUNNING_MODE != 'ONLY_GENOME':
tot_reads_aligned = df_summary_quantification['Reads_aligned'].fillna(0).sum()
tot_reads = df_summary_quantification['Reads_total'].sum()
if RUNNING_MODE=='AMPLICONS_AND_GENOME':
this_bam_filename = bam_filename_genome
if RUNNING_MODE=='ONLY_AMPLICONS':
this_bam_filename = bam_filename_amplicons
#if less than 1/2 of reads aligned, find most common unaligned reads and advise the user
if tot_reads > 0 and tot_reads_aligned/tot_reads < 0.5:
warn('Less than half (%d/%d) of reads aligned. Finding most frequent unaligned reads.'%(tot_reads_aligned,tot_reads))
###
###this results in the unpretty messages being printed:
### sort: write failed: standard output: Broken pipe
### sort: write error
###
#cmd = "samtools view -f 4 %s | awk '{print $10}' | sort | uniq -c | sort -nr | head -n 10"%this_bam_filename
import signal
def default_sigpipe():
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
cmd = "samtools view -f 4 %s | head -n 10000 | awk '{print $10}' | sort | uniq -c | sort -nr | head -n 10 | awk '{print $2}'"%this_bam_filename
# print("command is: "+cmd)
# p = sb.Popen(cmd, shell=True,stdout=sb.PIPE)
p = sb.Popen(cmd, shell=True,stdout=sb.PIPE,preexec_fn=default_sigpipe)
top_unaligned = p.communicate()[0]
top_unaligned_filename=_jp('CRISPRessoPooled_TOP_UNALIGNED.txt')
with open(top_unaligned_filename,'w') as outfile:
outfile.write(top_unaligned)
warn('Perhaps one or more of the given amplicon sequences were incomplete or incorrect. Below is a list of the most frequent unaligned reads (in the first 10000 unaligned reads). Check this list to see if an amplicon is among these reads.\n%s'%top_unaligned)
#cleaning up
if not args.keep_intermediate:
info('Removing Intermediate files...')
if args.fastq_r2!='':
files_to_remove=[processed_output_filename,flash_hist_filename,flash_histogram_filename,\
flash_not_combined_1_filename,flash_not_combined_2_filename]
else:
files_to_remove=[processed_output_filename]
if args.trim_sequences and args.fastq_r2!='':
files_to_remove+=[output_forward_paired_filename,output_reverse_paired_filename,\
output_forward_unpaired_filename,output_reverse_unpaired_filename]
if RUNNING_MODE=='ONLY_GENOME' or RUNNING_MODE=='AMPLICONS_AND_GENOME':
files_to_remove+=[bam_filename_genome]
if RUNNING_MODE=='ONLY_AMPLICONS':
files_to_remove+=[bam_filename_amplicons,amplicon_fa_filename]
for bowtie2_file in glob.glob(_jp('CUSTOM_BOWTIE2_INDEX.*')):
files_to_remove.append(bowtie2_file)
for file_to_remove in files_to_remove:
try:
if os.path.islink(file_to_remove):
#print 'LINK',file_to_remove
os.unlink(file_to_remove)
else:
os.remove(file_to_remove)
except:
warn('Skipping:%s' %file_to_remove)
info('All Done!')
print CRISPRessoShared.get_crispresso_footer()
sys.exit(0)
except Exception as e:
debug_flag = False
if 'args' in vars() and 'debug' in args:
debug_flag = args.debug
if debug_flag:
traceback.print_exc(file=sys.stdout)
error('\n\nERROR: %s' % e)
sys.exit(-1)
def main():
try:
description = [
'~~~CRISPRessoCompare~~~',
'-Comparison of two CRISPResso analyses-'
]
compare_header = r'''
___________________________
| __ __ __ __ __ |
|/ / \|\/||__) /\ |__)|_ |
|\__\__/| || /--\| \ |__ |
|___________________________|
'''
compare_header = CRISPRessoShared.get_crispresso_header(
description, compare_header)
print(compare_header)
parser = argparse.ArgumentParser(
description='CRISPRessoCompare Parameters',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'crispresso_output_folder_1',
type=str,
help='First output folder with CRISPResso analysis')
parser.add_argument(
'crispresso_output_folder_2',
type=str,
help='Second output folder with CRISPResso analysis')
#OPTIONALS
parser.add_argument('-n', '--name', help='Output name', default='')
parser.add_argument('-n1', '--sample_1_name', help='Sample 1 name')
parser.add_argument('-n2', '--sample_2_name', help='Sample 2 name')
parser.add_argument('-o', '--output_folder', help='', default='')
parser.add_argument(
'--min_frequency_alleles_around_cut_to_plot',
type=float,
help=
'Minimum %% reads required to report an allele in the alleles table plot.',
default=0.2)
parser.add_argument(
'--max_rows_alleles_around_cut_to_plot',
type=int,
help='Maximum number of rows to report in the alleles table plot. ',
default=50)
parser.add_argument('--suppress_report',
help='Suppress output report',
action='store_true')
parser.add_argument(
'--place_report_in_output_folder',
help=
'If true, report will be written inside the CRISPResso output folder. By default, the report will be written one directory up from the report output.',
action='store_true')
parser.add_argument('--debug',
help='Show debug messages',
action='store_true')
args = parser.parse_args()
debug_flag = args.debug
#check that the CRISPResso output is present and fill amplicon_info
quantification_file_1, amplicon_names_1, amplicon_info_1 = CRISPRessoShared.check_output_folder(
args.crispresso_output_folder_1)
quantification_file_2, amplicon_names_2, amplicon_info_2 = CRISPRessoShared.check_output_folder(
args.crispresso_output_folder_2)
run_info_1_file = os.path.join(args.crispresso_output_folder_1,
'CRISPResso2_info.pickle')
if os.path.isfile(run_info_1_file) is False:
raise CRISPRessoShared.OutputFolderIncompleteException(
'The folder %s is not a valid CRISPResso2 output folder. Cannot find run data at %s'
% (args.crispresso_output_folder_1, run_info_1_file))
run_info_1 = cp.load(open(run_info_1_file, 'rb'))
run_info_2_file = os.path.join(args.crispresso_output_folder_2,
'CRISPResso2_info.pickle')
if os.path.isfile(run_info_2_file) is False:
raise CRISPRessoShared.OutputFolderIncompleteException(
'The folder %s is not a valid CRISPResso2 output folder. Cannot find run data at %s'
% (args.crispresso_output_folder_2, run_info_2_file))
run_info_2 = cp.load(open(run_info_2_file, 'rb'))
sample_1_name = args.sample_1_name
if args.sample_1_name is None:
sample_1_name = "Sample 1"
if 'name' in run_info_1 and run_info_1['name'] != '':
sample_1_name = run_info_1['name']
sample_2_name = args.sample_2_name
if args.sample_2_name is None:
sample_2_name = "Sample 2"
if 'name' in run_info_2 and run_info_2['name'] != '':
sample_2_name = run_info_2['name']
get_name_from_folder = lambda x: os.path.basename(os.path.abspath(
x)).replace('CRISPResso_on_', '')
if not args.name:
database_id = '%s_VS_%s' % (
get_name_from_folder(args.crispresso_output_folder_1),
get_name_from_folder(args.crispresso_output_folder_2))
else:
database_id = args.name
OUTPUT_DIRECTORY = 'CRISPRessoCompare_on_%s' % database_id
if args.output_folder:
OUTPUT_DIRECTORY = os.path.join(
os.path.abspath(args.output_folder), OUTPUT_DIRECTORY)
_jp = lambda filename: os.path.join(
OUTPUT_DIRECTORY, filename
) #handy function to put a file in the output directory
log_filename = _jp('CRISPRessoCompare_RUNNING_LOG.txt')
try:
info('Creating Folder %s' % OUTPUT_DIRECTORY)
os.makedirs(OUTPUT_DIRECTORY)
info('Done!')
except:
warn('Folder %s already exists.' % OUTPUT_DIRECTORY)
log_filename = _jp('CRISPRessoCompare_RUNNING_LOG.txt')
logging.getLogger().addHandler(logging.FileHandler(log_filename))
with open(log_filename, 'w+') as outfile:
outfile.write(
'[Command used]:\nCRISPRessoCompare %s\n\n[Execution log]:\n' %
' '.join(sys.argv))
crispresso2Compare_info_file = os.path.join(
OUTPUT_DIRECTORY, 'CRISPResso2Compare_info.pickle')
crispresso2_info = {
} #keep track of all information for this run to be pickled and saved at the end of the run
crispresso2_info['version'] = CRISPRessoShared.__version__
crispresso2_info['args'] = deepcopy(args)
crispresso2_info['log_filename'] = os.path.basename(log_filename)
crispresso2_info['summary_plot_names'] = []
crispresso2_info['summary_plot_titles'] = {}
crispresso2_info['summary_plot_labels'] = {}
crispresso2_info['summary_plot_datas'] = {}
save_png = True
if args.suppress_report:
save_png = False
#LOAD DATA
amplicon_names_in_both = [
amplicon_name for amplicon_name in amplicon_names_1
if amplicon_name in amplicon_names_2
]
n_refs = len(amplicon_names_in_both)
def get_plot_title_with_ref_name(plotTitle, refName):
if n_refs > 1:
return (plotTitle + ": " + refName)
return plotTitle
for amplicon_name in amplicon_names_in_both:
profile_1 = parse_profile(
amplicon_info_1[amplicon_name]['quantification_file'])
profile_2 = parse_profile(
amplicon_info_2[amplicon_name]['quantification_file'])
amplicon_plot_name = amplicon_name + "."
if len(amplicon_names_in_both
) == 1 and amplicon_name == "Reference":
amplicon_plot_name = ""
try:
assert np.all(profile_1[:, 0] == profile_2[:, 0])
except:
raise DifferentAmpliconLengthException(
'Different amplicon lengths for the two amplicons.')
len_amplicon = profile_1.shape[0]
effect_vector_any_1 = profile_1[:, 1]
effect_vector_any_2 = profile_2[:, 1]
cut_points = run_info_1['refs'][amplicon_name]['sgRNA_cut_points']
sgRNA_intervals = run_info_1['refs'][amplicon_name][
'sgRNA_intervals']
#Quantification comparison barchart
fig = plt.figure(figsize=(30, 15))
n_groups = 2
N_TOTAL_1 = float(amplicon_info_1[amplicon_name]['Reads_aligned'])
N_UNMODIFIED_1 = float(
amplicon_info_1[amplicon_name]['Unmodified'])
N_MODIFIED_1 = float(amplicon_info_1[amplicon_name]['Modified'])
N_TOTAL_2 = float(amplicon_info_2[amplicon_name]['Reads_aligned'])
N_UNMODIFIED_2 = float(
amplicon_info_2[amplicon_name]['Unmodified'])
N_MODIFIED_2 = float(amplicon_info_2[amplicon_name]['Modified'])
means_sample_1 = np.array([N_UNMODIFIED_1, N_MODIFIED_1
]) / N_TOTAL_1 * 100
means_sample_2 = np.array([N_UNMODIFIED_2, N_MODIFIED_2
]) / N_TOTAL_2 * 100
ax1 = fig.add_subplot(1, 2, 1)
index = np.arange(n_groups)
bar_width = 0.35
opacity = 0.4
error_config = {'ecolor': '0.3'}
rects1 = ax1.bar(index,
means_sample_1,
bar_width,
alpha=opacity,
color=(0, 0, 1, 0.4),
label=sample_1_name)
rects2 = ax1.bar(index + bar_width,
means_sample_2,
bar_width,
alpha=opacity,
color=(1, 0, 0, 0.4),
label=sample_2_name)
plt.ylabel('% Sequences')
plt.title(
get_plot_title_with_ref_name(
'%s VS %s' % (sample_1_name, sample_2_name),
amplicon_name))
plt.xticks(index + bar_width / 2.0, ('Unmodified', 'Modified'))
plt.legend()
# plt.xlim(index[0]-0.2,(index + bar_width)[-1]+bar_width+0.2)
plt.tight_layout()
ax2 = fig.add_subplot(1, 2, 2)
ax2.bar(index,
means_sample_1 - means_sample_2,
bar_width + 0.35,
alpha=opacity,
color=(0, 1, 1, 0.4),
label='')
plt.ylabel('% Sequences Difference')
plt.title(
get_plot_title_with_ref_name(
'%s - %s' % (sample_1_name, sample_2_name), amplicon_name))
plt.xticks(index, ['Unmodified', 'Modified'])
# plt.xlim(index[0]-bar_width/2, (index+bar_width)[-1]+2*bar_width)
plt.tight_layout()
plot_name = '1.' + amplicon_plot_name + 'Editing_comparison'
plt.savefig(_jp(plot_name) + '.pdf', bbox_inches='tight')
if save_png:
plt.savefig(_jp(plot_name) + '.png', bbox_inches='tight')
crispresso2_info['summary_plot_names'].append(plot_name)
crispresso2_info['summary_plot_titles'][
plot_name] = 'Editing efficiency comparison'
crispresso2_info['summary_plot_labels'][
plot_name] = 'Figure 1: Comparison for amplicon ' + amplicon_name + '; Left: Percentage of modified and unmodified reads in each sample; Right: relative percentage of modified and unmodified reads'
output_1 = os.path.join(args.crispresso_output_folder_1,
run_info_1['report_filename'])
output_2 = os.path.join(args.crispresso_output_folder_1,
run_info_2['report_filename'])
crispresso2_info['summary_plot_datas'][plot_name] = []
if os.path.isfile(output_1):
crispresso2_info['summary_plot_datas'][plot_name].append(
(sample_1_name + ' output',
os.path.relpath(output_1, OUTPUT_DIRECTORY)))
if os.path.isfile(output_2):
crispresso2_info['summary_plot_datas'][plot_name].append(
(sample_2_name + ' output',
os.path.relpath(output_2, OUTPUT_DIRECTORY)))
mod_file_1 = amplicon_info_1[amplicon_name][
'modification_count_file']
amp_seq_1, mod_freqs_1 = CRISPRessoShared.parse_count_file(
mod_file_1)
mod_file_2 = amplicon_info_2[amplicon_name][
'modification_count_file']
amp_seq_2, mod_freqs_2 = CRISPRessoShared.parse_count_file(
mod_file_2)
consensus_sequence = amp_seq_1
if amp_seq_2 != consensus_sequence:
raise DifferentAmpliconLengthException(
'Different amplicon lengths for the two amplicons.')
for mod in [
'Insertions', 'Deletions', 'Substitutions',
'All_modifications'
]:
mod_name = mod
if mod == "All_modifications":
mod_name = "Combined modifications (insertions, deletions and substitutions)"
mod_counts_1 = np.array(mod_freqs_1[mod], dtype=float)
tot_counts_1 = np.array(mod_freqs_1['Total'], dtype=float)
unmod_counts_1 = tot_counts_1 - mod_counts_1
mod_counts_2 = np.array(mod_freqs_2[mod], dtype=float)
tot_counts_2 = np.array(mod_freqs_2['Total'], dtype=float)
unmod_counts_2 = tot_counts_2 - mod_counts_2
fisher_results = [
stats.fisher_exact([[z[0], z[1]], [z[2], z[3]]])
if max(z) > 0 else [nan, 1.0]
for z in zip(mod_counts_1, unmod_counts_1, mod_counts_2,
unmod_counts_2)
]
oddsratios, pvalues = [a for a, b in fisher_results
], [b for a, b in fisher_results]
mod_df = []
row = [sample_1_name + '_' + mod]
row.extend(mod_counts_1)
mod_df.append(row)
row = [sample_1_name + '_total']
row.extend(tot_counts_1)
mod_df.append(row)
row = [sample_2_name + '_' + mod]
row.extend(mod_counts_2)
mod_df.append(row)
row = [sample_2_name + '_total']
row.extend(tot_counts_2)
mod_df.append(row)
row = ['odds_ratios']
row.extend(oddsratios)
mod_df.append(row)
row = ['pvalues']
row.extend(pvalues)
mod_df.append(row)
colnames = ['Reference']
colnames.extend(list(consensus_sequence))
mod_df = pd.DataFrame(mod_df, columns=colnames)
# mod_df = pd.concat([mod_df.iloc[:,0:2], mod_df.iloc[:,2:].apply(pd.to_numeric)],axis=1)
#write to file
mod_filename = _jp(amplicon_plot_name + mod +
"_quantification.txt")
mod_df.to_csv(mod_filename, sep='\t', index=None)
#plot
fig = plt.figure(figsize=(20, 10))
ax1 = fig.add_subplot(2, 1, 1)
diff = np.divide(mod_counts_1, tot_counts_1) - np.divide(
mod_counts_2, tot_counts_2)
diff_plot = ax1.plot(diff,
color=(0, 1, 0, 0.4),
lw=3,
label='Difference')
ax1.set_title(
get_plot_title_with_ref_name(
'%s: %s - %s' % (mod, sample_1_name, sample_2_name),
amplicon_name))
ax1.set_xticks(
np.arange(
0, len_amplicon,
max(3, (len_amplicon / 6) -
(len_amplicon / 6) % 5)).astype(int))
ax1.set_ylabel('Sequences Difference %')
ax1.set_xlim(xmin=0, xmax=len_amplicon - 1)
pvalues = np.array(pvalues)
min_nonzero = np.min(pvalues[np.nonzero(pvalues)])
pvalues[pvalues == 0] = min_nonzero
#ax2 = ax1.twinx()
ax2 = fig.add_subplot(2, 1, 2)
pval_plot = ax2.plot(-1 * np.log10(pvalues),
color=(1, 0, 0, 0.4),
lw=2,
label='-log10 P-value')
ax2.set_ylabel('-log10 P-value')
ax2.set_xlim(xmin=0, xmax=len_amplicon - 1)
ax2.set_xticks(
np.arange(
0, len_amplicon,
max(3, (len_amplicon / 6) -
(len_amplicon / 6) % 5)).astype(int))
ax2.set_xlabel('Reference amplicon position (bp)')
#bonferroni correction
corrected_p = -1 * np.log10(
0.01 / float(len(consensus_sequence)))
cutoff_plot = ax2.plot([0, len(consensus_sequence)],
[corrected_p, corrected_p],
color='k',
dashes=(5, 10),
label='Bonferronni corrected cutoff')
plots = diff_plot + pval_plot + cutoff_plot
diff_y_min, diff_y_max = ax1.get_ylim()
p_y_min, p_y_max = ax2.get_ylim()
if cut_points:
for idx, cut_point in enumerate(cut_points):
if idx == 0:
plot_cleavage = ax1.plot(
[cut_point, cut_point],
[diff_y_min, diff_y_max],
'--k',
lw=2,
label='Predicted cleavage position')
ax2.plot([cut_point, cut_point],
[p_y_min, p_y_max],
'--k',
lw=2,
label='Predicted cleavage position')
plots = plots + plot_cleavage
else:
ax1.plot([cut_point, cut_point],
[diff_y_min, diff_y_max],
'--k',
lw=2,
label='_nolegend_')
ax2.plot([cut_point, cut_point],
[diff_y_min, diff_y_max],
'--k',
lw=2,
label='_nolegend_')
for idx, sgRNA_int in enumerate(sgRNA_intervals):
if idx == 0:
p2 = ax1.plot([sgRNA_int[0], sgRNA_int[1]],
[diff_y_min, diff_y_min],
lw=10,
c=(0, 0, 0, 0.15),
label='sgRNA')
ax2.plot([sgRNA_int[0], sgRNA_int[1]],
[p_y_min, p_y_min],
lw=10,
c=(0, 0, 0, 0.15),
label='sgRNA')
plots = plots + p2
else:
ax1.plot([sgRNA_int[0], sgRNA_int[1]],
[diff_y_min, diff_y_min],
lw=10,
c=(0, 0, 0, 0.15),
label='_nolegend_')
ax2.plot([sgRNA_int[0], sgRNA_int[1]],
[p_y_min, p_y_min],
lw=10,
c=(0, 0, 0, 0.15),
label='_nolegend_')
labs = [p.get_label() for p in plots]
lgd = plt.legend(plots,
labs,
loc='upper center',
bbox_to_anchor=(0.5, -0.2),
ncol=1,
fancybox=True,
shadow=False)
plot_name = '2.' + amplicon_plot_name + mod + '_quantification'
plt.savefig(_jp(plot_name + '.pdf'),
bbox_inches='tight',
bbox_extra_artists=(lgd, ))
if save_png:
plt.savefig(_jp(plot_name + '.png'),
bbox_inches='tight',
bbox_extra_artists=(lgd, ))
crispresso2_info['summary_plot_names'].append(plot_name)
crispresso2_info['summary_plot_titles'][
plot_name] = mod_name + ' locations'
crispresso2_info['summary_plot_labels'][
plot_name] = mod_name + ' location comparison for amplicon ' + amplicon_name + '; Top: percent difference; Bottom: p-value.'
crispresso2_info['summary_plot_datas'][plot_name] = [
(mod_name + ' quantification',
os.path.basename(mod_filename))
]
#create merged heatmaps for each cut site
allele_files_1 = amplicon_info_1[amplicon_name]['allele_files']
allele_files_2 = amplicon_info_2[amplicon_name]['allele_files']
for allele_file_1 in allele_files_1:
allele_file_1_name = os.path.split(allele_file_1)[
1] #get file part of path
for allele_file_2 in allele_files_2:
allele_file_2_name = os.path.split(allele_file_2)[
1] #get file part of path
#if files are the same (same amplicon, cut site, guide), run comparison
if allele_file_1_name == allele_file_2_name:
df1 = pd.read_csv(allele_file_1, sep="\t")
df2 = pd.read_csv(allele_file_2, sep="\t")
#find unmodified reference for comparison (if it exists)
ref_seq_around_cut = ""
if len(df1.loc[df1['Reference_Sequence'].str.contains(
'-') == False]) > 0:
ref_seq_around_cut = df1.loc[
df1['Reference_Sequence'].str.contains('-') ==
False]['Reference_Sequence'].iloc[0]
#otherwise figure out which sgRNA was used for this comparison
elif len(df2.loc[df2['Reference_Sequence'].str.
contains('-') == False]) > 0:
ref_seq_around_cut = df2.loc[
df2['Reference_Sequence'].str.contains('-') ==
False]['Reference_Sequence'].iloc[0]
else:
seq_len = df2[df2['Unedited'] ==
True]['Reference_Sequence'].iloc[0]
for sgRNA_interval, cut_point in zip(
sgRNA_intervals, cut_points):
sgRNA_seq = consensus_sequence[
sgRNA_interval[0]:sgRNA_interval[1]]
if sgRNA_seq in allele_file_1_name:
this_sgRNA_seq = sgRNA_seq
this_cut_point = cut_point
ref_seq_around_cut = consensus_sequence[max(
0, this_cut_point -
args.offset_around_cut_to_plot +
1):min(
len(reference_seq), cut_point +
args.offset_around_cut_to_plot +
1)]
break
merged = pd.merge(df1,
df2,
on=[
'Aligned_Sequence',
'Reference_Sequence', 'Unedited',
'n_deleted', 'n_inserted',
'n_mutated'
],
suffixes=('_' + sample_1_name,
'_' + sample_2_name),
how='outer')
quant_cols = [
'#Reads_' + sample_1_name,
'%Reads_' + sample_1_name,
'#Reads_' + sample_2_name,
'%Reads_' + sample_2_name
]
merged[quant_cols] = merged[quant_cols].fillna(0)
lfc_error = 0.1
merged['each_LFC'] = np.log2(
((merged['%Reads_' + sample_1_name] + lfc_error) /
(merged['%Reads_' + sample_2_name] + lfc_error)
).astype(float)).replace([np.inf, np.NaN], 0)
merged = merged.reset_index().set_index(
'Aligned_Sequence')
output_root = allele_file_1_name.replace(".txt", "")
allele_comparison_file = _jp(output_root + '.txt')
merged.to_csv(allele_comparison_file,
sep="\t",
index=None)
plot_name = '3.' + output_root + '_top'
CRISPRessoPlot.plot_alleles_table_compare(
ref_seq_around_cut,
merged.sort_values(['each_LFC'], ascending=True),
sample_1_name,
sample_2_name,
_jp(plot_name),
MIN_FREQUENCY=args.
min_frequency_alleles_around_cut_to_plot,
MAX_N_ROWS=args.
max_rows_alleles_around_cut_to_plot,
SAVE_ALSO_PNG=save_png)
crispresso2_info['summary_plot_names'].append(
plot_name)
crispresso2_info['summary_plot_titles'][
plot_name] = 'Alleles enriched in ' + sample_1_name
crispresso2_info['summary_plot_labels'][plot_name] = 'Distribution comparison of alleles. Nucleotides are indicated by unique colors (A = green; C = red; G = yellow; T = purple). Substitutions are shown in bold font. Red rectangles highlight inserted sequences. Horizontal dashed lines indicate deleted sequences. The vertical dashed line indicates the predicted cleavage site. '+ \
'The proportion and number of reads is shown for each sample on the right, with the values for ' + sample_1_name + ' followed by the values for ' + sample_2_name +'. Alleles are sorted for enrichment in ' + sample_1_name+'.'
crispresso2_info['summary_plot_datas'][plot_name] = [
('Allele comparison table',
os.path.basename(allele_comparison_file))
]
plot_name = '3.' + output_root + '_bottom'
CRISPRessoPlot.plot_alleles_table_compare(
ref_seq_around_cut,
merged.sort_values(['each_LFC'], ascending=False),
sample_1_name,
sample_2_name,
_jp(plot_name),
MIN_FREQUENCY=args.
min_frequency_alleles_around_cut_to_plot,
MAX_N_ROWS=args.
max_rows_alleles_around_cut_to_plot,
SAVE_ALSO_PNG=save_png)
crispresso2_info['summary_plot_names'].append(
plot_name)
crispresso2_info['summary_plot_titles'][
plot_name] = 'Alleles enriched in ' + sample_2_name
crispresso2_info['summary_plot_labels'][plot_name] = 'Distribution comparison of alleles. Nucleotides are indicated by unique colors (A = green; C = red; G = yellow; T = purple). Substitutions are shown in bold font. Red rectangles highlight inserted sequences. Horizontal dashed lines indicate deleted sequences. The vertical dashed line indicates the predicted cleavage site. '+ \
'The proportion and number of reads is shown for each sample on the right, with the values for ' + sample_1_name + ' followed by the values for ' + sample_2_name +'. Alleles are sorted for enrichment in ' + sample_2_name+'.'
crispresso2_info['summary_plot_datas'][plot_name] = [
('Allele comparison table',
os.path.basename(allele_comparison_file))
]
if not args.suppress_report:
if (args.place_report_in_output_folder):
report_name = _jp("CRISPResso2Batch_report.html")
else:
report_name = OUTPUT_DIRECTORY + '.html'
CRISPRessoReport.make_compare_report_from_folder(
report_name, crispresso2_info, OUTPUT_DIRECTORY, _ROOT)
crispresso2_info['report_location'] = report_name
crispresso2_info['report_filename'] = os.path.basename(report_name)
cp.dump(crispresso2_info, open(crispresso2Compare_info_file, 'wb'))
info('Analysis Complete!')
print(CRISPRessoShared.get_crispresso_footer())
sys.exit(0)
except Exception as e:
debug_flag = False
if 'args' in vars() and 'debug' in args:
debug_flag = args.debug
if debug_flag:
traceback.print_exc(file=sys.stdout)
error('\n\nERROR: %s' % e)
sys.exit(-1)
