def load_human_data(dataset_id):
if 'CSNVL' not in dataset_id:
lib_nm = _data.get_lib_nm(dataset_id)
lib_design, seq_col = _data.get_lib_design(dataset_id)
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
else:
# Use any conds to load 12kChar, CtoT, and AtoG libs
dids = ['190418_mES_12kChar_AID', '190329_HEK293T_AtoG_ABE', '190307_HEK_CtoT_BE4']
nms, seqs = [], []
for did in dids:
lib_design, seq_col = _data.get_lib_design(did)
nms += list(lib_design['Name (unique)'])
seqs += list(lib_design[seq_col])
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
Y_dir = _config.OUT_PLACE + 'combin_data_Y_imputewt/'
with gzip.open(Y_dir + '%s.pkl.gz' % (dataset_id), 'rb') as f:
Y = pickle.load(f)
NAMES = list(Y.keys())
Y = list(Y.values())
# Load X
if 'CSNVL' not in dataset_id:
zero_idxs = [_data.pos_to_idx(0, dataset_id)] * len(NAMES)
else:
zero_idxs = []
for nm in NAMES:
if 'satmut' in nm:
# 21
zero_idxs.append(_data.zero_pos['12kChar'])
else:
# CtoT = AtoG = 10
zero_idxs.append(_data.zero_pos['CtoT'])
X = []
timer = _util.Timer(total = len(NAMES))
for nm, y, zero_idx in zip(NAMES, Y, zero_idxs):
seq = nm_to_seq[nm]
# seq_30nt = seq[zero_idx - 9 : zero_idx + 20 + 1]
if zero_idx >= 9 + 10:
# 12kChar
pass
else:
# CtoT, AtoG libs
prefix = 'GATGGGTGCGACGCGTCAT'
seq = prefix + seq
zero_idx += len(prefix)
seq_50nt = seq[zero_idx - 9 - 10 : zero_idx + 20 + 10 + 1]
assert len(seq_50nt) == 50
X.append(seq_50nt)
return X, Y, NAMES
def adjust_treatment_control(treat_nm, control_nm, start_idx, end_idx):
adj_d = _data.load_data(treat_nm, 'ah6a1a_hf_bc')
control_data = _data.load_data(control_nm, 'h6_anyindel')
lib_design, seq_col = _data.get_lib_design(treat_nm)
lib_design = lib_design.iloc[start_idx:end_idx + 1]
nms = lib_design['Name (unique)']
'''
'''
print('Subtracting control from treatment data...')
shared_nms = [nm for nm in nms if nm in adj_d]
new_adj_d = dict()
timer = util.Timer(total=len(shared_nms))
for nm in shared_nms:
t = adj_d[nm]
if nm not in control_data:
continue
c = control_data[nm]
t = subtract_treatment_control(t, c)
new_adj_d[nm] = t
timer.update()
##
# Write
##
with open(out_dir + '%s_%s_%s.pkl' % (treat_nm, start_idx, end_idx),
'wb') as f:
pickle.dump(new_adj_d, f)
return
def remove_batch_effects(treat_nm, start_idx, end_idx):
batch_nm = exp_nm_to_batch[treat_nm]
lib_design, seq_col = _data.get_lib_design(treat_nm)
lib_nm = _data.get_lib_nm(treat_nm)
lib_design = lib_design.iloc[start_idx:end_idx + 1]
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
adj_d = _data.load_data(treat_nm, 'ah6a1b_subtract')
batch_muts_to_remove = pd.read_csv(
inp_dir + 'removed_batch_effects_%s.csv' % (lib_nm), index_col=0)
if len(batch_muts_to_remove) == 0:
inp_pkl = _config.OUT_PLACE + f'ah6a1b_subtract/{treat_nm}_{start_idx}_{end_idx}.pkl'
out_pkl = out_dir + f'{treat_nm}_{start_idx}_{end_idx}.pkl'
command = f'cp {inp_pkl} {out_pkl}'
subprocess.check_output(command, shell=True)
return
# Remove mutations
to_remove = batch_muts_to_remove[batch_muts_to_remove['Batch'] == batch_nm]
to_remove = to_remove[to_remove['Name'].isin(nms)]
adj_d = filter_mutations(to_remove, adj_d, nm_to_seq, treat_nm)
with open(out_dir + '%s_%s_%s.pkl' % (treat_nm, start_idx, end_idx),
'wb') as f:
pickle.dump(adj_d, f)
return
def gather_statistics(exp_nm, params):
(muts, allowed_pos, feature_radius) = params
# Load data
data = pd.read_csv(inp_dir + '%s.csv' % (exp_nm), index_col = 0)
# Set up library info
lib_nm = _data.get_lib_nm(exp_nm)
lib_design, seq_col = _data.get_lib_design(exp_nm)
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
# Prepare data
data = data[data['Total count'] >= 100]
data['Frequency'] = data['Count'] / data['Total count']
ontarget_sites = _data.get_ontarget_sites(lib_design, lib_nm)
data = data[data['Name'].isin(ontarget_sites)]
data = data[data['Position'].isin(allowed_pos)]
data['Mutation'] = data['Ref nt'] + '_' + data['Obs nt']
data['MutName'] = data['Name'].astype(str) + '_' + data['Position'].astype(str) + '_' + data['Mutation']
# Annotate with local sequence context
lib_zero_idx = _data.pos_to_idx(0, exp_nm)
dd = defaultdict(list)
print('Annotating data with local sequence contexts...')
timer = util.Timer(total = len(data))
for idx, row in data.iterrows():
seq = nm_to_seq[row['Name']]
pidx = row['Position'] + lib_zero_idx
local_context = seq[pidx - feature_radius : pidx] + seq[pidx + 1 : pidx + feature_radius + 1]
dd['Local context'].append(local_context)
timer.update()
for col in dd:
data[col] = dd[col]
# # Gather statistics
for mut_nm in muts:
print(mut_nm)
mut = muts[mut_nm]
if len(mut) == 1:
d_temp = data[data['Mutation'] == mut[0]]
else:
d_temp = data[data['Mutation'].isin(mut)]
d_temp['Mutation'] = mut_nm
d_temp['MutName'] = d_temp['Name'].astype(str) + '_' + d_temp['Position'].astype(str) + '_' + d_temp['Mutation']
group_cols = [s for s in d_temp.columns if s not in ['Frequency', 'Obs nt', 'Ref nt', 'Count']]
d_temp = d_temp.groupby(group_cols)['Frequency'].agg('sum').reset_index()
for ml_task in ['classify_zero', 'regress_nonzero']:
print(ml_task)
results = train_models(exp_nm, d_temp, mut_nm, ml_task)
save_results(exp_nm, mut_nm, ml_task, results)
return
def indel_anyindel(exp_nm):
try:
data = _data.load_data(exp_nm, 'ah6c_reduce_1bp_indel_fq')
except:
print('Error : could not load data')
sys.exit(1)
lib_design, seq_col = _data.get_lib_design(exp_nm)
dd = defaultdict(list)
timer = util.Timer(total = len(data))
for target_nm in data:
df = data[target_nm]
tot_count = sum(df['Count'])
dd['Total count'].append(tot_count)
dd['Name'].append(target_nm)
crit = (df['Category'] != 'wildtype')
indel_count = sum(df[crit]['Count'])
dd['Indel count'].append(indel_count)
if tot_count != 0:
dd['Indel freq'].append(indel_count / tot_count)
else:
dd['Indel freq'].append(np.nan)
crit = (df['Category'] == 'del')
del_count = sum(df[crit]['Count'])
dd['Del count'].append(del_count)
if tot_count != 0:
dd['Del freq'].append(del_count / tot_count)
else:
dd['Del freq'].append(np.nan)
crit = (df['Category'] == 'ins')
ins_count = sum(df[crit]['Count'])
dd['Ins count'].append(ins_count)
if tot_count != 0:
dd['Ins freq'].append(ins_count / tot_count)
else:
dd['Ins freq'].append(np.nan)
crit = (df['Category'] == 'wildtype')
wt_count = sum(df[crit]['Count'])
dd['Wildtype count'].append(wt_count)
if tot_count != 0:
dd['Wildtype freq'].append(wt_count / tot_count)
else:
dd['Wildtype freq'].append(np.nan)
timer.update()
df = pd.DataFrame(dd)
df.to_csv(out_dir + '%s.csv' % (exp_nm))
return
def adjust_batch_effects(exp_nm, start_idx, end_idx):
'''
Identify batch effects from position-wise analysis and remove them from combinatorial df
'''
lib_design, seq_col = _data.get_lib_design(exp_nm)
lib_design = lib_design.iloc[start_idx:end_idx + 1]
nms = lib_design['Name (unique)']
batch_muts = pd.read_csv(_config.DATA_DIR + 'batch_effects.csv')
batch = exp_nm_to_batch[exp_nm]
df = batch_muts[batch_muts['Batch'] == batch]
batch_cols, obs_nts = [], []
for idx, row in df.iterrows():
batch_col = '%s%s' % (row['Ref nt'], row['Position'])
batch_cols.append(batch_col)
obs_nts.append(row['Obs nt'])
data = pickle.load(open(inp_dir + '%s.pkl' % (exp_nm), 'rb'))
new_data = dict()
nms_shared = [nm for nm in nms if nm in data]
timer = util.Timer(total=len(nms_shared))
for target_nm in nms_shared:
d = data[target_nm]
d = d[d['Count'] > 0]
if 'index' in d.columns:
d = d[[col for col in d.columns if col != 'index']]
'''
Note: .loc is SUPER SLOW. Can be 100x slower than using iterrows
'''
# for batch_col, obs_nt in zip(batch_cols, obs_nts):
# if batch_col in d.columns:
# d.loc[d[batch_col] == obs_nt, batch_col] = '.'
matched_cols, matched_obs_nts = [], []
for idx, col in enumerate(batch_cols):
if col in d.columns:
matched_cols.append(col)
matched_obs_nts.append(obs_nts[idx])
for idx, row in d.iterrows():
for col, obs_nt in zip(matched_cols, matched_obs_nts):
if row[col] == obs_nt:
row[col] = '.'
new_data[target_nm] = d
timer.update()
with open(out_dir + '%s_%s_%s.pkl' % (exp_nm, start_idx, end_idx),
'wb') as f:
pickle.dump(new_data, f)
return
def form_L2_group_ae_newgenotype_Cas9_adjust(group_nm, l1_nms):
datas = [
_data.load_data(l1_nm, 'ae_newgenotype_Cas9_adjust')
for l1_nm in l1_nms
]
datas = [s for s in datas if s is not None]
lib_design, seq_col = _data.get_lib_design(l1_nms[0])
'''
g5 format: data is a dict, keys = target site names
values = dfs, with columns as '%s%s' % (nt, position), and 'Count' column
'''
group_ds = dict()
timer = util.Timer(total=len(lib_design))
for idx, row in lib_design.iterrows():
nm = row['Name (unique)']
timer.update()
# num_present = sum([bool(nm in data) for data in datas])
'''
Combine: two strategies
1. Normalize readcount, then add (equal contribution)
2. Directly add (weighted by readcount)
* using this strategy
'''
group_d = None
for data in datas:
if nm not in data:
continue
d = data[nm]
d = d.drop(columns=['_Sequence Context', '_Cutsite', '_ExpDir'])
nt_cols = [s for s in d.columns if s != 'Count']
d = d.fillna(value='.')
if group_d is None:
group_d = d
else:
group_d = group_d.append(d, ignore_index=True)
group_d = group_d.groupby(nt_cols)['Count'].sum()
group_d = group_d.reset_index()
if group_d is not None:
group_d = group_d.sort_values(by='Count', ascending=False)
group_d = group_d.reset_index()
group_d = group_d.replace(to_replace='.', value=np.nan)
group_ds[nm] = group_d
with open(out_dir + '%s.pkl' % (group_nm), 'wb') as f:
pickle.dump(group_ds, f)
return
def indel_anyindel_seq(exp_nm):
'''
Investigate if 1 nt deletions at abasic site are related to microhomology
Control for position by focusing only on pos 5
'''
df = pd.read_csv(inp_dir + '%s.csv' % (exp_nm), index_col = 0)
lib_design, seq_col = _data.get_lib_design(exp_nm)
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
dd = defaultdict(list)
all_nms = set(df['Name'])
five_idx = _data.pos_to_idx(4, exp_nm)
import code; code.interact(local=dict(globals(), **locals()))
timer = util.Timer(total = len(all_nms))
for nm in all_nms:
dfs = df[df['Name'] == nm]
seq = nm_to_seq[nm]
accept = bool(seq[five_idx] == 'C') & (seq[five_idx + 1] != 'C')
for jdx in range(five_idx - 1, -1, -1):
if seq[jdx] != 'C':
break
num_c = abs(five_idx - jdx)
if not accept:
continue
crit = (dfs['Category'] == 'del') & (dfs['Indel length'] == 1) & (dfs['Indel end adj'] == 5.0)
row = dfs[crit]
if len(row) == 0:
dd['Frequency'].append(0)
else:
dd['Frequency'].append(sum(row['Frequency']))
dd['Num C'].append(num_c)
dd['Name'].append(nm)
timer.update()
df = pd.DataFrame(dd)
df.to_csv(out_dir + '%s.csv' % (exp_nm))
return
def profile_subset(exp_nm, start_idx, end_idx):
editor_nm = exp_nm_to_editor[exp_nm]
muts_dict = get_editor_mutations(editor_nm)
editor_mut_set = set(muts_dict.keys())
'''
Identify batch effects from position-wise analysis and remove them from combinatorial df
'''
lib_design, seq_col = _data.get_lib_design(exp_nm)
lib_design = lib_design.iloc[start_idx:end_idx + 1]
nms = lib_design['Name (unique)']
data = pickle.load(open(inp_dir + '%s.pkl' % (exp_nm), 'rb'))
new_data = dict()
nms_shared = [nm for nm in nms if nm in data]
timer = util.Timer(total=len(nms_shared))
for target_nm in nms_shared:
d = data[target_nm]
d = d[d['Count'] > 0]
if 'index' in d.columns:
d = d[[col for col in d.columns if col != 'index']]
# Subset to mutation columns
_mut_cols = [col for col in d.columns if col in editor_mut_set]
d = d[_mut_cols + ['Count']]
for idx, row in d.iterrows():
for col in _mut_cols:
ref_nt = col[0]
obs_nt = row[col]
if obs_nt != ref_nt and obs_nt not in muts_dict[col]:
row[col] = '.'
# Eliminate rows that only contain .
crit = d.apply(
lambda row: sum([row[c] == '.'
for c in row.index]) != len(row.index) - 1,
axis='columns')
d = d[crit]
d = d.reset_index(drop=True)
new_data[target_nm] = d
timer.update()
with open(out_dir + '%s_%s_%s.pkl' % (exp_nm, start_idx, end_idx),
'wb') as f:
pickle.dump(new_data, f)
return
def indel_anyindel_seq(exp_nm):
'''
Annotate indels with related sequence context (e.g., bases in deletions)
'''
df = pd.read_csv(inp_dir + '%s.csv' % (exp_nm), index_col=0)
lib_design, seq_col = _data.get_lib_design(exp_nm)
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
indel_dd = defaultdict(list)
all_nms = set(df['Name'])
timer = util.Timer(total=len(df))
for idx, row in df.iterrows():
# dfs = df[df['Name'] == nm]
nm = row['Name']
seq = nm_to_seq[nm]
left_del_nt = np.nan
right_del_nt = np.nan
del_nts = np.nan
if row['Category'] == 'del':
start_pos = int(row['Indel start adj'])
start_idx = _data.pos_to_idx(start_pos, exp_nm)
end_pos = int(row['Indel end adj'])
end_idx = _data.pos_to_idx(end_pos, exp_nm)
if start_idx >= 0 and end_idx <= len(seq):
del_nts = seq[start_idx:end_idx]
left_del_nt = del_nts[0]
right_del_nt = del_nts[-1]
indel_dd['Left del nt'].append(left_del_nt)
indel_dd['Right del nt'].append(right_del_nt)
indel_dd['Del nts'].append(del_nts)
timer.update()
for col in indel_dd:
df[col] = indel_dd[col]
df.to_csv(out_dir + '%s.csv' % (exp_nm))
return
def form_L2_group_ag4_poswise_be_adjust(group_nm, l1_nms):
datas = [
_data.load_data(l1_nm, 'ag4_poswise_be_adjust') for l1_nm in l1_nms
]
datas = [s for s in datas if s is not None]
lib_design, seq_col = _data.get_lib_design(l1_nms[0])
'''
g4 format: data is a dict, keys = target site names
values = np.array with shape = (target site len, 4)
entries = int for num. Q30 observations
'''
group_ds = dict()
timer = util.Timer(total=len(lib_design))
for idx, row in lib_design.iterrows():
nm = row['Name (unique)']
timer.update()
# num_present = sum([bool(nm in data) for data in datas])
'''
Combine: two strategies
1. Normalize readcount, then add (equal contribution)
2. Directly add (weighted by readcount)
* using this strategy
'''
group_d = None
for data in datas:
if nm not in data:
continue
d = data[nm]
if group_d is None:
group_d = d
else:
group_d += d
if group_d is not None:
group_ds[nm] = group_d
with open(out_dir + '%s.pkl' % (group_nm), 'wb') as f:
pickle.dump(group_ds, f)
return
def gather_statistics(exp_nm):
# Load data
data = pd.read_csv(inp_dir +
'_batch_adjusted_all_ratios-ps0_1bpcorrect.csv',
index_col=0)
data = data[data['Condition'] == exp_nm]
# Set up library info
lib_nm = _data.get_lib_nm(exp_nm)
lib_design, seq_col = _data.get_lib_design(exp_nm)
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
ontarget_sites = _data.get_ontarget_sites(lib_design, lib_nm)
data = data[data['Name (unique)'].isin(ontarget_sites)]
# Annotate with local sequence context
# lib_zero_idx = _data.pos_to_idx(0, exp_nm)
dd = defaultdict(list)
print('Annotating data with local sequence contexts...')
timer = util.Timer(total=len(data))
for idx, row in data.iterrows():
seq = nm_to_seq[row['Name (unique)']]
lib_zero_idx = _data.pos_to_idx_safe(0, exp_nm, row['Name (unique)'])
# local_context = row['gRNA (20nt)']
local_context = seq[lib_zero_idx - 9:lib_zero_idx + 20 + 1]
dd['Local context'].append(local_context)
timer.update()
for col in dd:
data[col] = dd[col]
print(data.shape)
results = train_models(exp_nm, data,
'Log10 batch-adjusted base edit to indel ratio')
save_results(exp_nm, results)
return
def adjust_treatment_control(treat_nm, control_nm):
cas_data = _data.load_data(treat_nm, 'e_newgenotype_Cas9')
lib_data = _data.load_data(control_nm, 'e_newgenotype_Cas9')
cas_data = cas_data.drop(columns='ULMI count')
lib_data = lib_data.drop(columns='ULMI count')
lib_design, seq_col = _data.get_lib_design(treat_nm)
adj_d = dict()
stats_dd = defaultdict(list)
timer = util.Timer(total=len(lib_design))
for idx, row in lib_design.iterrows():
nm = row['Name (unique)']
timer.update()
cas = cas_data[cas_data['_Experiment'] == nm]
lib = lib_data[lib_data['_Experiment'] == nm]
stats_dd['Name'].append(nm)
if len(cas) == 0:
stats_dd['Status'].append('No treatment')
continue
if len(lib) == 0:
stats_dd['Status'].append('No control')
adj_d[nm] = t
continue
stats_dd['Status'].append('Adjusted')
new_cas = build_new_cas(lib, cas)
adj_d[nm] = new_cas
with open(out_dir + '%s.pkl' % (treat_nm), 'wb') as f:
pickle.dump(adj_d, f)
stats_df = pd.DataFrame(stats_dd)
stats_df.to_csv(out_dir + '%s_stats.csv' % (treat_nm))
return
def adjust_treatment_control(treat_nm, control_nm, start_idx, end_idx):
adj_d = _data.load_data(treat_nm, 'ag5a1a_hf_bc')
control_data = _data.load_data(control_nm, 'g5_combin_be')
lib_design, seq_col = _data.get_lib_design(treat_nm)
lib_design = lib_design.iloc[start_idx:end_idx + 1]
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
'''
g5 format: data is a dict, keys = target site names
values = dfs, with columns as '%s%s' % (nt, position), and 'Count' column
'''
'''
'''
print('Subtracting control from treatment data...')
shared_nms = [nm for nm in nms if nm in adj_d]
new_adj_d = dict()
timer = util.Timer(total=len(shared_nms))
for nm in shared_nms:
t = adj_d[nm]
if nm not in control_data:
continue
c = control_data[nm]
seq = nm_to_seq[nm]
t = subtract_treatment_control(t, c, seq)
new_adj_d[nm] = t
timer.update()
##
# Write
##
with open(out_dir + '%s_%s_%s.pkl' % (treat_nm, start_idx, end_idx),
'wb') as f:
pickle.dump(new_adj_d, f)
return
def gather_statistics(celltype, lib_nm, editor_nm):
print(celltype, lib_nm, editor_nm)
[rep1, rep2] = _data.get_replicates(celltype, lib_nm, editor_nm)
df1 = pd.read_csv(inp_dir + '%s.csv' % (rep1), index_col=0)
df2 = pd.read_csv(inp_dir + '%s.csv' % (rep2), index_col=0)
lib_nm = _data.get_lib_nm(rep1)
lib_design, seq_col = _data.get_lib_design(rep1)
ontarget_sites = _data.get_ontarget_sites(lib_design, lib_nm)
# Prepare data
# data = data[data['Total count'] >= 100]
df1 = df1[df1['Name (unique)'].isin(ontarget_sites)]
df2 = df2[df2['Name (unique)'].isin(ontarget_sites)]
id_cols = [
'Name (unique)',
'gRNA (20nt)',
seq_col,
]
mdf = df1.merge(df2, on=id_cols, suffixes=['_r1', '_r2'])
stat_col = 'Fraction edited'
mdf['absdiff'] = np.abs(mdf['%s_r1' % (stat_col)] - mdf['%s_r2' %
(stat_col)])
mdf['abslfc'] = np.abs(
np.log2(mdf['%s_r1' % (stat_col)]) - np.log2(mdf['%s_r2' %
(stat_col)]))
n_col = 'Total count'
mdf['Total n'] = mdf['%s_r1' % (n_col)] + mdf['%s_r2' % (n_col)]
mdf.to_csv(out_dir + '%s_%s_%s.csv' % (celltype, lib_nm, editor_nm))
return
def adjust_treatment_control(treat_nm, control_nm):
adj_d = _data.load_data(treat_nm, 'ag5a1b_subtract')
control_data = _data.load_data(control_nm, 'g5_combin_be')
treat_minq = _data.load_minq(treat_nm, 'g5_combin_be')
control_minq = _data.load_minq(control_nm, 'g5_combin_be')
lib_design, seq_col = _data.get_lib_design(treat_nm)
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
'''
g5 format: data is a dict, keys = target site names
values = dfs, with columns as '%s%s' % (nt, position), and 'Count' column
'''
'''
Filter treatment mutations that are best explained by
spontaneous random mutations.
Tend to be very low frequency with no counterpart in control
'''
print(
'Gathering statistics on treatment mutations explained by Illumina sequencing errors...'
)
ie_decisions = defaultdict(list)
timer = util.Timer(total=len(adj_d))
for nm in adj_d:
t = adj_d[nm]
if nm not in control_data:
continue
c = control_data[nm]
seq = nm_to_seq[nm]
c_minq = control_minq[nm]
t_minq = treat_minq[nm]
gather_stats_illumina_errors(t, c, t_minq, c_minq, seq, treat_nm, nm,
ie_decisions)
timer.update()
ie_fdr_threshold = 0.05
ie_df = pd.DataFrame(ie_decisions)
other_distribution = ie_df[ie_df['pval'] > 0.995]
ie_df = ie_df[ie_df['pval'] <= 0.995]
ie_df = ie_df.sort_values(by='pval')
ie_df = ie_df.reset_index(drop=True)
fdr_decs, hit_reject = [], False
for idx, pval in enumerate(ie_df['pval']):
if hit_reject:
dec = False
else:
fdr_critical = ((idx + 1) / len(ie_df)) * ie_fdr_threshold
dec = bool(pval <= fdr_critical)
fdr_decs.append(dec)
if dec is False and hit_reject is True:
hit_reject = False
ie_df['FDR accept'] = fdr_decs
other_distribution['FDR accept'] = False
ie_df = ie_df.append(other_distribution, ignore_index=True)
ie_df.to_csv(out_dir + '%s_ie_dec.csv' % (treat_nm))
print(
'Filtering treatment mutations explained by Illumina sequencing errors...'
)
to_remove = ie_df[ie_df['FDR accept'] == False]
adj_d = filter_illumina_error_muts(to_remove, adj_d, control_data,
nm_to_seq)
##
# Write
##
with open(out_dir + '%s.pkl' % (treat_nm), 'wb') as f:
pickle.dump(adj_d, f)
return
def adjust_treatment_control(treat_nm, control_nm):
treat_data = _data.load_data(treat_nm, 'h6_anyindel')
control_data = _data.load_data(control_nm, 'h6_anyindel')
lib_design, seq_col = _data.get_lib_design(treat_nm)
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
'''
h6 format: data is a dict, keys = target site names
values = dfs
'Category',
'Indel start',
'Indel end',
'Indel length',
'MH length',
'Inserted bases',
'Count',
'Name',
'''
adj_d = dict()
stats_dd = defaultdict(list)
'''
Filter positions with abnormally high control mut freq.
'''
hc_decisions = defaultdict(list)
print('Filtering positions with high frequency control mutations...')
timer = util.Timer(total = len(lib_design))
for idx, row in lib_design.iterrows():
timer.update()
nm = row['Name (unique)']
seq = row[seq_col]
stats_dd['Name'].append(nm)
if nm not in treat_data:
stats_dd['Status'].append('No treatment')
continue
t = treat_data[nm]
if nm not in control_data:
stats_dd['Status'].append('No control')
adj_d[nm] = t
continue
stats_dd['Status'].append('Adjusted')
c = control_data[nm]
# Adjust
t = filter_high_control_muts(t, c, seq, treat_nm, nm, hc_decisions)
if t is not None:
adj_d[nm] = t
stats_df = pd.DataFrame(stats_dd)
stats_df.to_csv(out_dir + '%s_stats.csv' % (treat_nm))
hc_df = pd.DataFrame(hc_decisions)
hc_df = hc_df.sort_values(by = 'max_hf_freq', ascending = False)
hc_df = hc_df.reset_index(drop = True)
hc_df.to_csv(out_dir + '%s_hc_dec.csv' % (treat_nm))
'''
Filter treatment mutations that can be explained by control freq.
In practice, this step is most effective for control mutations
with relatively high frequency => relatively high variance
'''
print('Gathering statistics on treatment mutations explained by control mutations...')
bc_decisions = defaultdict(list)
timer = util.Timer(total = len(adj_d))
for nm in adj_d:
t = adj_d[nm]
if nm not in control_data:
continue
c = control_data[nm]
seq = nm_to_seq[nm]
gather_stats_binom_control_muts(t, c, seq, treat_nm, nm, bc_decisions)
timer.update()
'''
Using global statistics, filter mutations
while controlling false discovery rate
'''
bc_fdr_threshold = 0.05
bc_df = pd.DataFrame(bc_decisions)
other_distribution = bc_df[bc_df['pval'] > 0.995]
bc_df = bc_df[bc_df['pval'] <= 0.995]
bc_df = bc_df.sort_values(by = 'pval')
bc_df = bc_df.reset_index(drop = True)
fdr_decs, hit_reject = [], False
for idx, pval in enumerate(bc_df['pval']):
if hit_reject:
dec = False
else:
fdr_critical = ((idx + 1) / len(bc_df)) * bc_fdr_threshold
dec = bool(pval <= fdr_critical)
fdr_decs.append(dec)
if dec is False and hit_reject is True:
hit_reject = False
bc_df['FDR accept'] = fdr_decs
other_distribution['FDR accept'] = False
bc_df = bc_df.append(other_distribution, ignore_index = True)
bc_df.to_csv(out_dir + '%s_bc_dec.csv' % (treat_nm))
print('Filtering treatment mutations explained by control mutations...')
to_remove = bc_df[bc_df['FDR accept'] == False]
adj_d = filter_binom_control_muts(to_remove, adj_d, control_data, nm_to_seq)
##
# Write
##
with open(out_dir + '%s.pkl' % (treat_nm), 'wb') as f:
pickle.dump(adj_d, f)
return
def adjust_treatment_control(treat_nm, control_nm):
treat_data = _data.load_data(treat_nm, 'g5_combin_be')
control_data = _data.load_data(control_nm, 'g5_combin_be')
treat_minq = _data.load_minq(treat_nm, 'g5_combin_be')
control_minq = _data.load_minq(control_nm, 'g5_combin_be')
lib_design, seq_col = _data.get_lib_design(treat_nm)
'''
g5 format: data is a dict, keys = target site names
values = dfs, with columns as '%s%s' % (nt, position), and 'Count' column
'''
adj_d = dict()
stats_dd = defaultdict(list)
hc_decisions = defaultdict(list)
nm_to_seq = dict()
'''
Filter positions with abnormally high control mut freq.
'''
print('Filtering positions with high frequency control mutations...')
timer = util.Timer(total = len(lib_design))
for idx, row in lib_design.iterrows():
nm = row['Name (unique)']
seq = row[seq_col]
nm_to_seq[nm] = seq
timer.update()
stats_dd['Name'].append(nm)
if nm not in treat_data:
stats_dd['Status'].append('No treatment')
continue
t = treat_data[nm]
if nm not in control_data:
stats_dd['Status'].append('No control')
adj_d[nm] = t
continue
stats_dd['Status'].append('Adjusted')
c = control_data[nm]
# Adjust
t = filter_high_control_muts(t, c, seq, treat_nm, nm, hc_decisions)
adj_d[nm] = t
stats_df = pd.DataFrame(stats_dd)
stats_df.to_csv(out_dir + '%s_stats.csv' % (treat_nm))
hc_df = pd.DataFrame(hc_decisions)
hc_df = hc_df.sort_values(by = 'c_fq', ascending = False)
hc_df = hc_df.reset_index(drop = True)
hc_df.to_csv(out_dir + '%s_hc_dec.csv' % (treat_nm))
'''
Filter treatment mutations that can be explained by control freq.
In practice, this step is most effective for control mutations
with relatively high frequency => relatively high variance
'''
print('Gathering statistics on treatment mutations explained by control mutations...')
bc_decisions = defaultdict(list)
timer = util.Timer(total = len(adj_d))
for nm in adj_d:
t = adj_d[nm]
if nm not in control_data:
continue
c = control_data[nm]
seq = nm_to_seq[nm]
gather_stats_binom_control_muts(t, c, seq, treat_nm, nm, bc_decisions)
timer.update()
'''
Using global statistics, filter mutations
while controlling false discovery rate
'''
bc_fdr_threshold = 0.05
bc_df = pd.DataFrame(bc_decisions)
other_distribution = bc_df[bc_df['pval'] > 0.995]
bc_df = bc_df[bc_df['pval'] <= 0.995]
bc_df = bc_df.sort_values(by = 'pval')
bc_df = bc_df.reset_index(drop = True)
fdr_decs, hit_reject = [], False
for idx, pval in enumerate(bc_df['pval']):
if hit_reject:
dec = False
else:
fdr_critical = ((idx + 1) / len(bc_df)) * bc_fdr_threshold
dec = bool(pval <= fdr_critical)
fdr_decs.append(dec)
if dec is False and hit_reject is True:
hit_reject = False
bc_df['FDR accept'] = fdr_decs
other_distribution['FDR accept'] = False
bc_df = bc_df.append(other_distribution, ignore_index = True)
bc_df.to_csv(out_dir + '%s_bc_dec.csv' % (treat_nm))
print('Filtering treatment mutations explained by control mutations...')
to_remove = bc_df[bc_df['FDR accept'] == False]
adj_d = filter_binom_control_muts(to_remove, adj_d, control_data, nm_to_seq)
'''
'''
print('Subtracting control from treatment data...')
timer = util.Timer(total = len(adj_d))
for nm in adj_d:
t = adj_d[nm]
if nm not in control_data:
continue
c = control_data[nm]
seq = nm_to_seq[nm]
t = subtract_treatment_control(t, c, seq)
adj_d[nm] = t
timer.update()
'''
Filter treatment mutations that are best explained by
spontaneous random mutations.
Tend to be very low frequency with no counterpart in control
'''
print('Gathering statistics on treatment mutations explained by Illumina sequencing errors...')
ie_decisions = defaultdict(list)
timer = util.Timer(total = len(adj_d))
for nm in adj_d:
t = adj_d[nm]
if nm not in control_data:
continue
c = control_data[nm]
seq = nm_to_seq[nm]
c_minq = control_minq[nm]
t_minq = treat_minq[nm]
gather_stats_illumina_errors(t, c, t_minq, c_minq, seq, treat_nm, nm, ie_decisions)
timer.update()
ie_fdr_threshold = 0.05
ie_df = pd.DataFrame(ie_decisions)
other_distribution = ie_df[ie_df['pval'] > 0.995]
ie_df = ie_df[ie_df['pval'] <= 0.995]
ie_df = ie_df.sort_values(by = 'pval')
ie_df = ie_df.reset_index(drop = True)
fdr_decs, hit_reject = [], False
for idx, pval in enumerate(ie_df['pval']):
if hit_reject:
dec = False
else:
fdr_critical = ((idx + 1) / len(ie_df)) * ie_fdr_threshold
dec = bool(pval <= fdr_critical)
fdr_decs.append(dec)
if dec is False and hit_reject is True:
hit_reject = False
ie_df['FDR accept'] = fdr_decs
other_distribution['FDR accept'] = False
ie_df = ie_df.append(other_distribution, ignore_index = True)
ie_df.to_csv(out_dir + '%s_ie_dec.csv' % (treat_nm))
print('Filtering treatment mutations explained by Illumina sequencing errors...')
to_remove = ie_df[ie_df['FDR accept'] == False]
adj_d = filter_illumina_error_muts(to_remove, adj_d, control_data, nm_to_seq)
##
# Write
##
with open(out_dir + '%s.pkl' % (treat_nm), 'wb') as f:
pickle.dump(adj_d, f)
return
def fig_editing_profiles(treat_nm):
'''
g4 format: data is a dict, keys = target site names
values = np.array with shape = (target site len, 4)
entries = int for num. Q30 observations
'''
adj_d = pickle.load(open(inp_dir + '%s.pkl' % (treat_nm), 'rb'))
lib_design, seq_col = _data.get_lib_design(treat_nm)
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
lib_nm = _data.get_lib_nm(treat_nm)
ontarget_nms = set(_data.get_ontarget_sites(lib_design, lib_nm))
'''
Filter treatment mutations that match the unedited background profile
using the statistic: fraction of target sites with non-zero event frequency
'''
print('Forming long df...')
dd = defaultdict(list)
timer = util.Timer(total = len(adj_d))
for nm in adj_d:
timer.update()
if nm not in ontarget_nms:
continue
pw = adj_d[nm]
seq = nm_to_seq[nm]
for jdx in range(len(pw)):
tot = np.nansum(pw[jdx])
ref_nt = seq[jdx]
ref_idx = nt_to_idx[ref_nt]
for kdx in range(len(pw[jdx])):
if kdx == ref_idx:
continue
count = pw[jdx][kdx]
dd['Count'].append(count)
dd['Total count'].append(tot)
dd['Obs nt'].append(nts[kdx])
dd['Ref nt'].append(ref_nt)
if tot == 0:
dd['Frequency'].append(np.nan)
else:
dd['Frequency'].append(count / tot)
dd['Position index'].append(jdx)
dd['Position'].append(_data.idx_to_pos(jdx, treat_nm))
dd['Name'].append(nm)
df = pd.DataFrame(dd)
df = df[df['Total count'] >= 100]
n_targetsites_in_condition = len(df)
# Form stats_df
dd = defaultdict(list)
pos_range = sorted(set(df['Position index']))
timer = util.Timer(total = len(pos_range))
for pos_idx in pos_range:
timer.update()
df_s1 = df[df['Position index'] == pos_idx]
for ref_nt in nts:
df_s2 = df_s1[df_s1['Ref nt'] == ref_nt]
for obs_nt in nts:
if obs_nt == ref_nt:
continue
crit = (df_s2['Obs nt'] == obs_nt)
dfs = df_s2[crit]
dfs_freq = dfs['Frequency']
num_zeros = sum(dfs_freq == 0)
total = len(dfs_freq)
if total == 0:
continue
dd['Num target sites with zero for mutation'].append(num_zeros)
dd['Total num target sites for mutation'].append(total)
dd['Frequency of zero in target sites for mutation'].append(num_zeros / total)
dd['Num target sites in condition'].append(n_targetsites_in_condition)
dd['Mean activity'].append(np.mean(dfs_freq))
dd['Position index'].append(pos_idx)
dd['Position'].append(_data.idx_to_pos(pos_idx, treat_nm))
dd['Obs nt'].append(obs_nt)
dd['Ref nt'].append(ref_nt)
hm_df = pd.DataFrame(dd)
hm_df.to_csv(out_dir + '%s.csv' % (treat_nm))
# Median normalize
background_range = range(25, 34 + 1)
for ref_nt in nts:
for obs_nt in nts:
if obs_nt == ref_nt:
continue
crit = (hm_df['Ref nt'] == ref_nt) & (hm_df['Obs nt'] == obs_nt) & (~np.isnan(hm_df['Mean activity']))
medi = np.nanmedian(hm_df[crit & (hm_df['Position'].isin(background_range))]['Mean activity'])
hm_df.loc[crit, 'Mean activity'] = hm_df.loc[crit, 'Mean activity'].apply(lambda x: max(0, x - medi))
hm_df.to_csv(out_dir + '%s_median_bg_adj.csv' % (treat_nm))
return
def gather_statistics(exp_nm):
feature_radius = 10
allowed_pos = range(3, 8 + 1)
# Load data
data = pd.read_csv(inp_dir + '%s.csv' % (exp_nm), index_col=0)
# Set up library info
lib_nm = _data.get_lib_nm(exp_nm)
lib_design, seq_col = _data.get_lib_design(exp_nm)
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
# Prepare data
data = data[data['Total count'] >= 100]
data['Frequency'] = data['Count'] / data['Total count']
ontarget_sites = _data.get_ontarget_sites(lib_design, lib_nm)
data = data[data['Name'].isin(ontarget_sites)]
data = data[data['Position'].isin(allowed_pos)]
data['Mutation'] = data['Ref nt'] + '_' + data['Obs nt']
# data['MutName'] = data['Name'].astype(str) + '_' + data['Position'].astype(str) + '_' + data['Mutation']
# Annotate with local sequence context
lib_zero_idx = _data.pos_to_idx(0, exp_nm)
dd = defaultdict(list)
print('Annotating data with local sequence contexts...')
timer = util.Timer(total=len(data))
for idx, row in data.iterrows():
seq = nm_to_seq[row['Name']]
pidx = row['Position'] + lib_zero_idx
local_context = seq[pidx -
feature_radius:pidx] + seq[pidx + 1:pidx +
feature_radius + 1]
dd['Local context'].append(local_context)
timer.update()
for col in dd:
data[col] = dd[col]
# # Gather statistics
# for mut_nm in muts:
# print(mut_nm)
# mut = muts[mut_nm]
# if len(mut) == 1:
# d_temp = data[data['Mutation'] == mut[0]]
# else:
# d_temp = data[data['Mutation'].isin(mut)]
# d_temp['Mutation'] = mut_nm
# d_temp['MutName'] = d_temp['Name'].astype(str) + '_' + d_temp['Position'].astype(str) + '_' + d_temp['Mutation']
# group_cols = [s for s in d_temp.columns if s not in ['Frequency', 'Obs nt', 'Ref nt', 'Count']]
# d_temp = d_temp.groupby(group_cols)['Frequency'].agg('sum').reset_index()
print(data.columns)
print(set(data['Mutation']))
acc_muts = [
'C_T',
'C_G',
'C_A',
]
data = data[data['Mutation'].isin(acc_muts)]
data = data.drop(columns=['Count', 'Total count', 'Ref nt', 'Obs nt'])
data = data.pivot_table(
index=['Name', 'Position', 'Local context'],
columns='Mutation',
values='Frequency',
).reset_index()
data = data.fillna(value=0)
numerator = data['C_G'] + data['C_A']
denominator = data['C_T'] + data['C_G'] + data['C_A']
data['Frequency'] = numerator / denominator
data = data.dropna()
mut_name = 'C_GA_over_C_D'
data['MutName'] = data['Name'].astype(str) + '_' + data['Position'].astype(
str) + '_' + mut_name
print(data.shape)
for ml_task in ['regress_nonzero', 'classify_zero']:
print(ml_task)
results = train_models(exp_nm, data, mut_name, ml_task)
save_results(exp_nm, mut_name, ml_task, results)
##
numerator = data['C_T']
denominator = data['C_T'] + data['C_G'] + data['C_A']
data['Frequency'] = numerator / denominator
data = data.dropna()
mut_name = 'C_T_over_C_D'
data['MutName'] = data['Name'].astype(str) + '_' + data['Position'].astype(
str) + '_' + mut_name
print(data.shape)
for ml_task in ['regress_nonzero', 'classify_zero']:
print(ml_task)
results = train_models(exp_nm, data, mut_name, ml_task)
save_results(exp_nm, mut_name, ml_task, results)
##
numerator = data['C_G']
denominator = data['C_T'] + data['C_G'] + data['C_A']
data['Frequency'] = numerator / denominator
data = data.dropna()
mut_name = 'C_G_over_C_D'
data['MutName'] = data['Name'].astype(str) + '_' + data['Position'].astype(
str) + '_' + mut_name
print(data.shape)
for ml_task in ['regress_nonzero', 'classify_zero']:
print(ml_task)
results = train_models(exp_nm, data, mut_name, ml_task)
save_results(exp_nm, mut_name, ml_task, results)
##
numerator = data['C_A']
denominator = data['C_T'] + data['C_G'] + data['C_A']
data['Frequency'] = numerator / denominator
data = data.dropna()
mut_name = 'C_A_over_C_D'
data['MutName'] = data['Name'].astype(str) + '_' + data['Position'].astype(
str) + '_' + mut_name
print(data.shape)
for ml_task in ['regress_nonzero', 'classify_zero']:
print(ml_task)
results = train_models(exp_nm, data, mut_name, ml_task)
save_results(exp_nm, mut_name, ml_task, results)
##
numerator = data['C_G']
denominator = data['C_A'] + data['C_G']
data['Frequency'] = numerator / denominator
data = data.dropna()
mut_name = 'C_G_over_C_GA'
data['MutName'] = data['Name'].astype(str) + '_' + data['Position'].astype(
str) + '_' + mut_name
print(data.shape)
for ml_task in ['regress_nonzero', 'classify_zero']:
print(ml_task)
results = train_models(exp_nm, data, mut_name, ml_task)
save_results(exp_nm, mut_name, ml_task, results)
return
def form_data(exp_nm, start_idx, end_idx):
'''
Annotate library design with total count, edited count, fraction edited, etc.
'''
data = _data.load_data(exp_nm, 'ag5a4_profile_subset')
lib_design, seq_col = _data.get_lib_design(exp_nm)
lib_nm = _data.get_lib_nm(exp_nm)
lib_design = lib_design.iloc[start_idx : end_idx + 1]
ontarget_sites = _data.get_ontarget_sites(lib_design, lib_nm)
lib_design = lib_design[lib_design['Name (unique)'].isin(ontarget_sites)]
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
stats_dd = defaultdict(list)
new_data = dict()
nms_shared = [nm for nm in nms if nm in data]
timer = util.Timer(total = len(nms_shared))
for iter, nm in enumerate(nms_shared):
df = data[nm]
seq = nm_to_seq[nm]
num_mismatches = lambda x, y: sum([bool(n1 != n2) for n1,n2 in zip(x,y)])
if 'index' in df.columns:
df = df[[col for col in df.columns if col != 'index']]
if len(df) == 0: continue
## 8/21/19
'''
Simulate bystander precision task in 12kChar by using the substrate nucleotide closest to the editor-specific center nt
'''
editor = _data.get_editor_nm(exp_nm)
editor_to_central_pos = {
'ABE': 6,
'ABE-CP': 6,
'AID': 6,
'BE4': 6,
'BE4-CP': 8,
'CDA': 5,
'eA3A': 6,
'evoAPOBEC': 5,
}
if editor in editor_to_central_pos:
central_pos = editor_to_central_pos[editor]
else:
central_pos = 6
substrate = 'A' if 'ABE' in editor else 'C'
nt_cols = [f'{substrate}{pos}' for pos in range(-3, 15) if f'{substrate}{pos}' in df.columns]
central_col = find_central_col(central_pos, nt_cols, substrate)
if central_col is None: continue
mut_cols = [col for col in df.columns if col != 'Count']
col_to_ref_nt = {col: col[0] for col in mut_cols}
df_dd = defaultdict(list)
for idx, row in df.iterrows():
df_dd['Num. edits'].append(get_num_edits(row, col_to_ref_nt))
df_dd['Simulated precise'].append(is_simulated_precise(row, central_col, col_to_ref_nt))
for col in df_dd:
df[col] = df_dd[col]
numer = sum(df[df['Simulated precise'] == True]['Count'])
denom = sum(df[df['Num. edits'] > 0]['Count'])
sim_precision = numer / denom if denom > 0 else np.nan
stats_dd['Simulated bystander precision at editor-specific central nt'].append(sim_precision)
stats_dd['Simulated bystander position'].append(int(central_col[1:]))
stats_dd['Simulated bystander position, distance to center'].append(int(central_col[1:]) - central_pos)
edited_ct = sum(df[df['Num. edits'] > 0]['Count'])
stats_dd['Edited count'].append(edited_ct)
stats_dd['Name (unique)'].append(nm)
timer.update()
stats_df_collected = pd.DataFrame(stats_dd)
stats_df = lib_design.merge(
stats_df_collected,
on = 'Name (unique)',
how = 'outer',
)
stats_df.to_csv(out_dir + '%s_%s_%s_stats.csv' % (exp_nm, start_idx, end_idx))
return
def form_data(exp_nm, start_idx, end_idx):
data = _data.load_data(exp_nm, 'ag5a4_profile_subset')
lib_design, seq_col = _data.get_lib_design(exp_nm)
lib_nm = _data.get_lib_nm(exp_nm)
disease_nms = _data.get_disease_sites(lib_design, lib_nm)
# Subset for dumb parallelization, ensure only disease target sites used
lib_design = lib_design.iloc[start_idx:end_idx + 1]
lib_design = lib_design[lib_design['Name (unique)'].isin(disease_nms)]
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
stats_dd = defaultdict(list)
nms_shared = [nm for nm in nms if nm in data]
timer = util.Timer(total=len(nms_shared))
for iter, nm in enumerate(nms_shared):
df = data[nm]
seq = nm_to_seq[nm]
design_row = lib_design[lib_design['Name (unique)'] == nm].iloc[0]
snp_pos = int(design_row['Position of SNP in gRNA'])
correct_nt = design_row['Corrected nucleotide (gRNA orientation)']
path_nt = design_row['Pathogenic nucleotide (gRNA orientation)']
nt_cols = [
col for col in df.columns if col != 'Count' and col != 'Frequency'
]
# Impute . as wildtype
df = impute_dot_as_wildtype(df, nt_cols)
total_ct = sum(df['Count'])
# Ensure each row is unique
df = df.groupby(nt_cols)['Count'].agg('sum').reset_index()
# Filter unedited columns
df = subset_edited_rows(df, nt_cols)
edited_ct = sum(df['Count'])
df = remove_noisy_edits(df, nt_cols, exp_nm)
gt_correct_ct = get_precise_gt_correction_count(
df, nt_cols, snp_pos, correct_nt, path_nt)
## Overall statistics
stats_dd['Name (unique)'].append(nm)
stats_dd['Obs. correction count'].append(gt_correct_ct)
stats_dd['Obs. total count'].append(total_ct)
stats_dd['Obs. edited count'].append(edited_ct)
stats_dd['Obs. gt correct fraction in all reads'].append(
gt_correct_ct / total_ct if total_ct > 0 else np.nan)
stats_dd['Obs. gt correct precision in edited reads'].append(
gt_correct_ct / edited_ct if edited_ct > 0 else np.nan)
stats_dd['Obs. editing frequency'].append(
edited_ct / total_ct if total_ct > 0 else np.nan)
# Amino acid correction for CtoGA
if 'AA sequence - reference' in design_row.index and type(
design_row['AA sequence - reference']) == str:
orients = list('-+')
d1 = bool(design_row['Designed orientation w.r.t. genome'] == '+')
d2 = bool(design_row['AA frame strand'] == '+')
xor_int = int(d1 == d2)
aa_strand_relative_to_seq = orients[xor_int]
aa_stats = {
'Unedited AA': 0,
'Edited AA': 0,
'Goal AA': 0,
}
if design_row['AA sequence - pathogenic'] != design_row[
'AA sequence - reference']:
for jdx, edit_row in df.iterrows():
seq_30nt = edit_row_to_seq_30nt(design_row, edit_row,
seq_col)
obs_aas = nts_to_aas(seq_30nt,
design_row['AA frame position'],
snp_pos, aa_strand_relative_to_seq)
pp0idx = design_row['Protospacer position zero index']
seq_30nt_path = design_row[seq_col][pp0idx - 9:pp0idx + 21]
aa_path_with_bc = nts_to_aas(
seq_30nt_path, design_row['AA frame position'],
snp_pos, aa_strand_relative_to_seq)
seq_30nt_wt = seq_30nt_path[:9 + snp_pos] + design_row[
'Corrected nucleotide (gRNA orientation)'] + seq_30nt_path[
9 + snp_pos + 1:]
aa_wt_with_bc = nts_to_aas(seq_30nt_wt,
design_row['AA frame position'],
snp_pos,
aa_strand_relative_to_seq)
if obs_aas == aa_path_with_bc:
aa_stats['Unedited AA'] += edit_row['Count']
else:
aa_stats['Edited AA'] += edit_row['Count']
if obs_aas == aa_wt_with_bc:
aa_stats['Goal AA'] += edit_row['Count']
stats_dd['Obs. aa correct precision among edited gts'].append(
aa_stats['Goal AA'] / edited_ct if edited_ct > 0 else np.nan)
stats_dd['Obs. aa correct precision among edited aas'].append(
aa_stats['Goal AA'] /
aa_stats['Edited AA'] if aa_stats['Edited AA'] > 0 else np.nan)
stats_dd['Obs. aa correct precision among all reads'].append(
aa_stats['Goal AA'] / total_ct if total_ct > 0 else np.nan)
if stats_dd[
'Obs. aa correct precision among edited gts'] < stats_dd[
'Obs. gt correct precision in edited reads']:
import code
code.interact(local=dict(globals(), **locals()))
else:
stats_dd['Obs. aa correct precision among edited gts'].append(
np.nan)
stats_dd['Obs. aa correct precision among edited aas'].append(
np.nan)
stats_dd['Obs. aa correct precision among all reads'].append(
np.nan)
timer.update()
# Save
stats_df_collected = pd.DataFrame(stats_dd)
stats_df = lib_design.merge(
stats_df_collected,
on='Name (unique)',
how='outer',
)
stats_df.to_csv(out_dir + '%s_%s_%s_stats.csv' %
(exp_nm, start_idx, end_idx))
return
def adjust_batch_effects(lib_nm):
print(lib_nm)
# Gather statistics
be_treatments = []
batch_set = set()
batch_to_exp_nms = defaultdict(list)
for treat_nm in treat_control_df['Treatment']:
if 'Cas9' in treat_nm:
continue
if _data.get_lib_nm(treat_nm) != lib_nm:
continue
batch_nm = exp_nm_to_batch[treat_nm]
be_treatments.append(treat_nm)
batch_set.add(batch_nm)
batch_to_exp_nms[batch_nm].append(treat_nm)
lib_design, seq_col = _data.get_lib_design(be_treatments[0])
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
md = dict()
timer = util.Timer(total=len(be_treatments))
print('Loading stats from each condition...')
for treat_nm in be_treatments:
with open(inp_dir + '%s.pkl' % (treat_nm), 'rb') as f:
d = pickle.load(f)
md[treat_nm] = d
# df['Treatment'] = treat_nm
# df['Batch'] = exp_nm_to_batch[treat_nm]
# df['Editor'] = exp_nm_to_editor[treat_nm]
timer.update()
# ANOVA calculations
from scipy.stats import f_oneway
print(
'Calculating ANOVA on all unique indels in all target sites to identify batch effects...'
)
dd = defaultdict(list)
means_dd = defaultdict(lambda: defaultdict(lambda: dict()))
timer = util.Timer(total=len(nms))
for exp_nm in nms:
mut_dd, all_mut_nms = form_dd(md, exp_nm)
for mut_nm in all_mut_nms:
anova_args = defaultdict(list)
# Note: Ensure we do not implicitly treat a lack of data as an observation of zero
for exp_nm_2 in mut_dd:
anova_args[exp_nm_to_batch[exp_nm_2]].append(
mut_dd[exp_nm_2][mut_nm])
'''
Ensure non-degenerate ANOVA testing.
If every batch has 0 std, we have identical values. It's likely that these identical values are 0 because of the sparsity of the data when considering unique indels (highly heterogeneous) at 12,000 target sites.
If every batch with a non-zero value has only one observation, skip.
'''
# Only perform ANOVA test on indels where at least one batch has non-zero std (otherwise it was seen only once in any batch, so it's not a batch effect)
num_non_zero_stds = 0
mean_d, std_d = dict(), dict()
for batch in batch_set:
if batch in anova_args:
mean_val = np.mean(anova_args[batch])
std_val = np.std(anova_args[batch])
if std_val > 0:
num_non_zero_stds += 1
else:
mean_val = np.nan
std_val = np.nan
mean_d[batch] = mean_val
std_d[batch] = std_val
degenerate_flag = False
if num_non_zero_stds == 0:
for batch in batch_set:
batch_data = anova_args[batch]
if len(batch_data) == 0:
continue
has_non_zero = bool(batch_data.count(0) != len(batch_data))
if has_non_zero and len(batch_data) == 1:
degenerate_flag = True
# elif has_non_zero and len(batch_data) > 1:
# import code; code.interact(local=dict(globals(), **locals()))
if degenerate_flag:
continue
aa = tuple([s for s in anova_args.values() if len(s) != 0])
if len(aa) < 2:
continue
fstat, pval = f_oneway(*aa)
if np.isnan(pval):
continue
dd['Statistic'].append(fstat)
dd['pval'].append(pval)
dd['MutName'].append(mut_nm)
dd['Name'].append(exp_nm)
for batch in batch_set:
dd['Mean %s' % (batch)].append(mean_d[batch])
dd['Std %s' % (batch)].append(std_d[batch])
means_dd[exp_nm][mut_nm][batch] = mean_val
timer.update()
stats_df = pd.DataFrame(dd)
if len(stats_df) == 0:
empty_df = pd.DataFrame()
empty_df.to_csv(out_dir + 'mutation_dec_%s.csv' % (lib_nm))
empty_df.to_csv(out_dir + 'removed_batch_effects_%s.csv' % (lib_nm))
empty_df.to_csv(out_dir + 'removed_stats_%s.csv' % (lib_nm))
return
stats_df['-log10p'] = -np.log10(stats_df['pval'])
# Apply FDR
print(
'Finding significant batch effects while controlling false discovery...'
)
fdr_threshold = 0.01
other_distribution = stats_df[stats_df['pval'] > 0.995]
stats_df = stats_df[stats_df['pval'] <= 0.995]
stats_df = stats_df.sort_values(by='pval')
stats_df = stats_df.reset_index(drop=True)
fdr_decs, hit_reject = [], False
for idx, pval in enumerate(stats_df['pval']):
if hit_reject:
dec = False
else:
fdr_critical = ((idx + 1) / len(stats_df)) * fdr_threshold
dec = bool(pval <= fdr_critical)
fdr_decs.append(dec)
if dec is False and hit_reject is True:
hit_reject = False
stats_df['FDR accept'] = fdr_decs
other_distribution['FDR accept'] = False
stats_df = stats_df.append(other_distribution, ignore_index=True)
stats_df.to_csv(out_dir + 'mutation_dec_%s.csv' % (lib_nm))
'''
Identify mutations for removal
At mutations passing Bonferroni corrected ANOVA test,
identify batches where mutations are frequent
'''
print('Identifying batches to remove mutations from...')
to_remove = stats_df[stats_df['FDR accept'] == True]
dd = defaultdict(list)
dd_stats = defaultdict(list)
timer = util.Timer(total=len(to_remove))
for idx, row in to_remove.iterrows():
timer.update()
exp_nm = row['Name']
mut_nm = row['MutName']
means = means_dd[exp_nm][mut_nm]
mean_vals = list(means.values())
mean_means = np.mean(mean_vals)
for batch_nm in means:
if means[batch_nm] >= mean_means or means[batch_nm] >= 0.005:
dd['Batch'].append(batch_nm)
dd['Name'].append(exp_nm)
dd['MutName'].append(mut_nm)
for batch_nm in means:
dd_stats['%s' % (batch_nm)].append(means[batch_nm])
dd_stats['MutName'].append(mut_nm)
dd_stats['Name'].append(exp_nm)
batch_muts_to_remove = pd.DataFrame(dd)
batch_muts_to_remove.to_csv(out_dir + 'removed_batch_effects_%s.csv' %
(lib_nm))
batch_muts_stats = pd.DataFrame(dd_stats)
batch_muts_stats.to_csv(out_dir + 'removed_stats_%s.csv' % (lib_nm))
# Mutations are removed in ah6a3
return
def indel_anyindel_pos(exp_nm):
data = None
if exp_nm in set(treat_control_df['Treatment']):
data = _data.load_data(exp_nm, 'ah6a3_remove_batch')
is_control = False
elif exp_nm in set(treat_control_df['Control']):
data = _data.load_data(exp_nm, 'h6_anyindel')
is_control = True
if data is None:
print('Error : could not load data')
import code
code.interact(local=dict(globals(), **locals()))
sys.exit(1)
lib_design, seq_col = _data.get_lib_design(exp_nm)
# Init
pos_dd = dict()
for pos_idx in range(-25, 50):
pos_dd[pos_idx] = []
pos_dd['Name'] = []
# Init
len_dd = dict()
for len_val in range(-40, 15 + 1):
len_dd[len_val] = []
len_dd['Name'] = []
# Init
pos_len_indel = dict()
for indel_len in range(-40, 15 + 1):
pos_nt_indel = dict()
for pos_idx in range(-25, 50):
pos_nt_indel[pos_idx] = []
pos_nt_indel['Name'] = []
pos_len_indel[indel_len] = pos_nt_indel
mdf = pd.DataFrame()
timer = util.Timer(total=len(data))
for target_nm in data:
df = data[target_nm]
if 'Frequency' not in df.columns:
df['Frequency'] = df['Count'] / np.sum(df['Count'])
tot_count = sum(df['Count'])
if tot_count < 100:
continue
crit = (df['Category'] != 'wildtype')
dfs = df[crit]
# Init
target_pos_vector = defaultdict(lambda: 0)
target_pos_len_vectors = dict()
for indel_len in range(-40, 15 + 1):
target_pos_vector_nt = defaultdict(lambda: 0)
target_pos_len_vectors[indel_len] = target_pos_vector_nt
indel_len_vector = defaultdict(lambda: 0)
# Iterate
df_annot = defaultdict(list)
for idx, row in dfs.iterrows():
indel_start = int(row['Indel start'])
indel_end = int(row['Indel end'])
mh_len = row['MH length']
freq = row['Frequency']
indel_len = int(row['Indel length'])
cat = row['Category']
# Gather indel length frequencies
if cat == 'del':
indel_len = indel_len * -1
elif cat == 'ins':
'''
h6_anyindel in each dir describes indel_start and indel_end for indexing (should have difference of 1 for insertions but they do not). so we fix this here
'''
indel_len = indel_len
indel_end = indel_start + 1
indel_len_vector[indel_len] += freq
# Adjust start and end
if not is_control:
package = adjust_indel_pos(indel_start, indel_end, mh_len)
(adj_indel_start, adj_indel_end) = package
else:
adj_indel_start, adj_indel_end = indel_start, indel_end
# Gather total frequency by position
for jdx in range(adj_indel_start, adj_indel_end):
target_pos_vector[jdx] += freq
# Gather total frequency by position of specific nt indels
if indel_len in target_pos_len_vectors:
target_pos_vector_nt = target_pos_len_vectors[indel_len]
for jdx in range(adj_indel_start, adj_indel_end):
target_pos_vector_nt[jdx] += freq
df_annot['Indel start adj'].append(adj_indel_start)
df_annot['Indel end adj'].append(adj_indel_end)
for col in df_annot:
dfs[col] = df_annot[col]
dfs['Name'] = target_nm
mdf = mdf.append(dfs, ignore_index=True)
# Gather indel length frequencies
for col in len_dd:
if col != 'Name':
len_dd[col].append(indel_len_vector[col])
else:
len_dd[col].append(target_nm)
# Gather total frequency by position
for col in pos_dd:
if col != 'Name':
pos_dd[col].append(target_pos_vector[col])
else:
pos_dd[col].append(target_nm)
# Gather total frequency by position of 1 nt indels
for indel_len in pos_len_indel:
pos_nt_indel = pos_len_indel[indel_len]
tpvn = target_pos_len_vectors[indel_len]
for col in pos_nt_indel:
if col != 'Name':
pos_nt_indel[col].append(tpvn[col])
else:
pos_nt_indel[col].append(target_nm)
timer.update()
# Save
pos_df = pd.DataFrame(pos_dd)
pos_df.to_csv(out_dir + '%s_pos.csv' % (exp_nm))
pos_df_melt = pd.melt(pos_df,
id_vars='Name',
var_name='Position',
value_name='Frequency')
pos_df_melt.to_csv(out_dir + '%s_pos_melt.csv' % (exp_nm))
# Save
for indel_len in pos_len_indel:
pos_nt_indel = pos_len_indel[indel_len]
pos_nt_df = pd.DataFrame(pos_nt_indel)
pos_nt_df.to_csv(out_dir + '%s_pos_%snt.csv' % (exp_nm, indel_len))
pos_nt_df_melt = pd.melt(pos_nt_df,
id_vars='Name',
var_name='Position',
value_name='Frequency')
pos_nt_df_melt.to_csv(out_dir + '%s_pos_melt_%snt.csv' %
(exp_nm, indel_len))
# Save
len_df = pd.DataFrame(len_dd)
len_df.to_csv(out_dir + '%s_len.csv' % (exp_nm))
len_df_melt = pd.melt(len_df,
id_vars='Name',
var_name='Indel length',
value_name='Frequency')
len_df_melt.to_csv(out_dir + '%s_len_melt.csv' % (exp_nm))
# merged
mdf.to_csv(out_dir + '%s.csv' % (exp_nm))
return
def calc_indels_global(exp_nm):
'''
Across all target sites, measure frequency of indels by length and starting position.
Allow low readcount cutoff, but also do not normalize data within each target site by total readcount.
'''
data = None
if exp_nm in set(treat_control_df['Treatment']):
data = _data.load_data(exp_nm, 'ah6a1b_subtract')
elif exp_nm in set(treat_control_df['Control']):
data = _data.load_data(exp_nm, 'h6_anyindel')
pass
if data is None:
print('Error : could not load data')
import code; code.interact(local=dict(globals(), **locals()))
sys.exit(1)
lib_design, seq_col = _data.get_lib_design(exp_nm)
# Init
pos_len_indel = dict()
for indel_len in range(-40, 15 + 1):
pos_nt_indel = dict()
for pos_idx in range(-25, 50):
pos_nt_indel[pos_idx] = []
pos_nt_indel['Name'] = []
pos_len_indel[indel_len] = pos_nt_indel
mdf = pd.DataFrame()
timer = util.Timer(total = len(data))
for target_nm in data:
df = data[target_nm]
if 'Frequency' not in df.columns:
df['Frequency'] = df['Count'] / np.sum(df['Count'])
tot_count = sum(df['Count'])
if tot_count < 100:
continue
crit = (df['Category'] != 'wildtype')
dfs = df[crit]
# Init
target_pos_len_vectors = dict()
for indel_len in range(-40, 15 + 1):
target_pos_vector_nt = defaultdict(lambda: 0)
target_pos_len_vectors[indel_len] = target_pos_vector_nt
indel_len_vector = defaultdict(lambda: 0)
# Iterate
df_annot = defaultdict(list)
for idx, row in dfs.iterrows():
indel_start = int(row['Indel start'])
indel_end = int(row['Indel end'])
mh_len = row['MH length']
count = row['Count']
indel_len = int(row['Indel length'])
cat = row['Category']
# Gather indel length frequencies
if cat == 'del':
indel_len = indel_len * -1
elif cat == 'ins':
'''
h6_anyindel in each dir describes indel_start and indel_end for indexing (should have difference of 1 for insertions but they do not). so we fix this here
'''
indel_len = indel_len
indel_end = indel_start + 1
indel_len_vector[indel_len] += count
# Adjust start and end
package = adjust_indel_pos(indel_start, indel_end, mh_len)
(adj_indel_start, adj_indel_end) = package
# Add count to adj. start pos at the right length
if indel_len in target_pos_len_vectors:
target_pos_vector_nt = target_pos_len_vectors[indel_len]
target_pos_vector_nt[adj_indel_start] += count
df_annot['Indel start adj'].append(adj_indel_start)
df_annot['Indel end adj'].append(adj_indel_end)
for col in df_annot:
dfs[col] = df_annot[col]
dfs['Name'] = target_nm
mdf = mdf.append(dfs, ignore_index = True)
# Gather total frequency by position and length of indels
for indel_len in pos_len_indel:
pos_nt_indel = pos_len_indel[indel_len]
tpvn = target_pos_len_vectors[indel_len]
for col in pos_nt_indel:
if col != 'Name':
pos_nt_indel[col].append(tpvn[col])
else:
pos_nt_indel[col].append(target_nm)
timer.update()
# Save
for indel_len in pos_len_indel:
pos_nt_indel = pos_len_indel[indel_len]
pos_nt_df = pd.DataFrame(pos_nt_indel)
pos_nt_df.to_csv(out_dir + '%s_pos_%snt.csv' % (exp_nm, indel_len))
pos_nt_df_melt = pd.melt(pos_nt_df, id_vars = 'Name', var_name = 'Position', value_name = 'Count')
pos_nt_df_melt.to_csv(out_dir + '%s_pos_melt_%snt.csv' % (exp_nm, indel_len))
# merged
mdf.to_csv(out_dir + '%s.csv' % (exp_nm))
return
def form_data(exp_nm):
data = _data.load_data(exp_nm, 'ag4_poswise_be_adjust')
lib_design, seq_col = _data.get_lib_design(exp_nm)
# Get target nt
editor_type = _data.get_editor_type(exp_nm)
if editor_type == 'CtoTeditor':
target_nt = 'C'
elif editor_type == 'AtoGeditor':
target_nt = 'A'
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
grnas = lib_design['gRNA (20nt)']
design_cats = lib_design['Design category']
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
nm_to_grna = {nm: grna for nm, grna in zip(nms, grnas)}
nm_to_design_cat = {
nm: design_cat
for nm, design_cat in zip(nms, design_cats)
}
dd = defaultdict(list)
timer = util.Timer(total=len(data))
for nm in data:
pw = data[nm]
seq = nm_to_seq[nm]
grna = nm_to_grna[nm]
design_cat = nm_to_design_cat[nm]
# Get category, subcategory, and match count
match_count = get_match_count(grna, seq)
if design_cat == 'guideseq':
category = 'Off-target series'
subcategory = nm.split('_')[2] # gene name
elif design_cat == 'mismatch':
category = 'Mismatch series'
subcategory = nm.split('_')[1] # series number
elif design_cat == 'chipseq':
category = 'Chip series'
elif design_cat == 'vivo':
category = 'vivo'
subcategory = 'vivo'
else:
assert match_count == 20, 'fail'
category = 'On-target'
subcategory = 'On-target'
for jdx in range(len(pw)):
pos = _data.idx_to_pos(jdx, exp_nm)
if pos not in [6, 7]:
continue
ref_nt = seq[jdx]
if ref_nt != target_nt:
continue
ref_idx = nt_to_idx[ref_nt]
total = sum(pw[jdx])
edit_ct = 0
for kdx in range(len(pw[jdx])):
if kdx == ref_idx:
continue
edit_ct += pw[jdx][kdx]
if total > 0:
dd['Edited fraction'].append(edit_ct / total)
else:
dd['Edited fraction'].append(np.nan)
dd['Edit count'].append(edit_ct)
dd['Total count'].append(total)
dd['Position'].append(pos)
dd['Ref nt'].append(ref_nt)
dd['Name'].append(nm)
dd['Match count'].append(int(match_count))
dd['Category'].append(category)
dd['Subcategory'].append(subcategory)
timer.update()
df = pd.DataFrame(dd)
df.to_csv(out_dir + '%s.csv' % (exp_nm))
return
def form_data(exp_nm, start_idx, end_idx):
'''
Annotate library design with total count, edited count, fraction edited, etc.
'''
data = _data.load_data(exp_nm, 'ag5a4_profile_subset')
lib_design, seq_col = _data.get_lib_design(exp_nm)
lib_design = lib_design.iloc[start_idx:end_idx + 1]
nms = lib_design['Name (unique)']
seqs = lib_design[seq_col]
nm_to_seq = {nm: seq for nm, seq in zip(nms, seqs)}
stats_dd = defaultdict(list)
new_data = dict()
nms_shared = [nm for nm in nms if nm in data]
timer = util.Timer(total=len(nms_shared))
for iter, nm in enumerate(nms_shared):
df = data[nm]
seq = nm_to_seq[nm]
num_mismatches = lambda x, y: sum(
[bool(n1 != n2) for n1, n2 in zip(x, y)])
if 'index' in df.columns:
df = df[[col for col in df.columns if col != 'index']]
if len(df) == 0:
continue
## Row-wise statistics
mut_cols = [col for col in df.columns if col != 'Count']
col_to_ref_nt = {col: col[0] for col in mut_cols}
df_dd = defaultdict(list)
for idx, row in df.iterrows():
df_dd['Num. edits'].append(get_num_edits(row, col_to_ref_nt))
df_dd['Has aberrant CBE edit'].append(
has_aberrant_cbe_edits(row, col_to_ref_nt))
for col in df_dd:
df[col] = df_dd[col]
new_data[nm] = df
## Overall statistics
stats_dd['Name (unique)'].append(nm)
edited_ct = sum(df[df['Num. edits'] > 0]['Count'])
stats_dd['Edited count'].append(edited_ct)
cbe_aberrant_ct = sum(df[df['Has aberrant CBE edit'] == True]['Count'])
stats_dd['CBE aberrant count'].append(cbe_aberrant_ct)
total_ct = sum(df['Count'])
stats_dd['Total count'].append(total_ct)
try:
frac = edited_ct / total_ct
except ZeroDivisionError:
frac = np.nan
stats_dd['Fraction edited'].append(frac)
try:
frac = cbe_aberrant_ct / total_ct
except ZeroDivisionError:
frac = np.nan
stats_dd['Fraction CBE aberrant edit'].append(frac)
timer.update()
# Save
with open(out_dir + '%s_%s_%s.pkl' % (exp_nm, start_idx, end_idx),
'wb') as f:
pickle.dump(new_data, f)
stats_df_collected = pd.DataFrame(stats_dd)
import code
code.interact(local=dict(globals(), **locals()))
stats_df = lib_design.merge(
stats_df_collected,
on='Name (unique)',
how='outer',
)
stats_df.to_csv(out_dir + '%s_%s_%s_stats.csv' %
(exp_nm, start_idx, end_idx))
return
def indel_anyindel(exp_nm):
try:
data = _data.load_data(exp_nm, 'ah6a1b_subtract')
except:
print('Error : could not load data')
sys.exit(1)
lib_design, seq_col = _data.get_lib_design(exp_nm)
dd = defaultdict(list)
timer = util.Timer(total=len(data))
for target_nm in data:
df = data[target_nm]
tot_count = sum(df['Count'])
dd['Total count'].append(tot_count)
dd['Name (unique)'].append(target_nm)
crit = (df['Category'] != 'wildtype')
indel_count = sum(df[crit]['Count'])
dd['Indel count'].append(indel_count)
if tot_count != 0:
dd['Indel freq'].append(indel_count / tot_count)
else:
dd['Indel freq'].append(np.nan)
crit = (df['Category'] == 'del')
del_count = sum(df[crit]['Count'])
dd['Del count'].append(del_count)
if tot_count != 0:
dd['Del freq'].append(del_count / tot_count)
else:
dd['Del freq'].append(np.nan)
crit = (df['Category'] == 'ins')
ins_count = sum(df[crit]['Count'])
dd['Ins count'].append(ins_count)
if tot_count != 0:
dd['Ins freq'].append(ins_count / tot_count)
else:
dd['Ins freq'].append(np.nan)
crit = (df['Category'] == 'wildtype')
wt_count = sum(df[crit]['Count'])
dd['Wildtype count'].append(wt_count)
if tot_count != 0:
dd['Wildtype freq'].append(wt_count / tot_count)
else:
dd['Wildtype freq'].append(np.nan)
timer.update()
df = pd.DataFrame(dd)
data = lib_design.merge(
df,
on='Name (unique)',
how='outer',
)
# Annotate csvs
pam_start_idx = 33
pam_len = 5
get_pam = lambda row: row['Sequence context (61nt)'][
pam_start_idx:pam_start_idx + pam_len]
data['Designed PAM (5nt)'] = data.apply(get_pam, axis='columns')
get_true_grna_len = lambda row: 20 if row['gRNA (20nt)'][0] == 'G' else 21
data['True gRNA length'] = data.apply(get_true_grna_len, axis='columns')
grna_pos1_idx = 13
grna_pos0_idx = grna_pos1_idx - 1
grna_5primeG_matches_target = lambda row: bool(row[
'Sequence context (61nt)'][grna_pos1_idx] == 'G') if bool(row[
'True gRNA length'] == 20) else bool(row['Sequence context (61nt)']
[grna_pos0_idx] == 'G')
data['gRNA 5primeG matches target'] = data.apply(
grna_5primeG_matches_target, axis='columns')
# Prepare data
# data = data[data['Total count'] >= 100]
# # Gather statistics
def grna_5primeg_and_len(row):
grna_len = row['True gRNA length']
match = row['gRNA 5primeG matches target']
if match:
return f'{grna_len}-nt gRNA, 5primeG matches'
else:
return f'{grna_len}-nt gRNA, 5primeG does not match'
data['gRNA properties'] = data.apply(grna_5primeg_and_len, axis='columns')
data.to_csv(out_dir + f'{exp_nm}.csv')
data = data[data['Total count'] >= 100]
pv_df = data.pivot(index='Name (unique)',
columns='gRNA properties',
values='Indel freq')
pv_df.to_csv(out_dir + f'5primeG_{exp_nm}.csv')
return
