def test_profile_ct_bincounts(self):
""" Test the ability of sortseq_tools.profile_ct to count frequencies
"""
print '\nIn test_profile_ct_bincounts...'
library_files = glob.glob(self.input_dir+'library_*.txt')
library_files += glob.glob(self.input_dir+'dataset_*.txt')
good_bin_num = 2
bad_bin_num = 5
for file_name in library_files:
print '\t%s ='%file_name,
description = file_name.split('_')[-1].split('.')[0]
executable = lambda:\
profile_ct.main(io.load_dataset(file_name),bin=good_bin_num)
print '(bin=%d)'%good_bin_num,
# If bad or library, then profile_ct.main should raise SortSeqError
if ('_bad' in file_name) or ('library' in file_name):
try:
self.assertRaises(SortSeqError,executable)
print 'badtype,',
except:
print 'good (ERROR).'
raise
# If good, then profile_ct.main should produce a valid df
elif ('_good' in file_name) or ('dataset' in file_name):
try:
df = executable()
qc.validate_profile_ct(df)
out_file = self.output_dir+\
'profile_ct_bin_%s.txt'%description
io.write(df,out_file)
io.load_profile_ct(out_file)
print 'good,',
except:
print 'bad (ERROR).'
raise
# There are no other options
else:
raise SortSeqError('Unrecognized class of file_name.')
# Should always raise an error if bin num is too large
executable = lambda:\
profile_ct.main(io.load_dataset(file_name),bin=bad_bin_num)
print '(bin=%d)'%bad_bin_num,
try:
self.assertRaises(SortSeqError,executable)
print 'badtype.'
except:
print 'good (ERROR).'
raise
def main(dataset_df, bin=None, start=0, end=None):
"""
Computes character frequencies (0.0 to 1.0) at each position
Arguments:
dataset_df (pd.DataFrame): A dataframe containing a valid dataset.
bin (int): A bin number specifying which counts to use
start (int): An integer specifying the sequence start position
end (int): An integer specifying the sequence end position
Returns:
freq_df (pd.DataFrame): A dataframe containing counts for each nucleotide/amino acid character at each position.
"""
# Validate dataset_df
qc.validate_dataset(dataset_df)
# Compute counts
counts_df = profile_ct.main(dataset_df, bin=bin, start=start, end=end)
# Create columns for profile_freqs table
ct_cols = [c for c in counts_df.columns if qc.is_col_type(c,'ct_')]
freq_cols = ['freq_'+c.split('_')[1] for c in ct_cols]
# Compute frequencies from counts
freq_df = counts_df[ct_cols].div(counts_df['ct'], axis=0)
freq_df.columns = freq_cols
freq_df['pos'] = counts_df['pos']
# Validate as counts dataframe
freq_df = qc.validate_profile_freq(freq_df,fix=True)
return freq_df
def main(dataset_df, bin=None, start=0, end=None, err=False):
"""
Computes the mutation rate (0.0 to 1.0) at each position. Mutation rate is defined as 1.0 minus the maximum character frequency at a position. Errors are estimated using bionomial uncertainty
Arguments:
dataset_df (pd.DataFrame): A dataframe containing a valid dataset.
bin (int): A bin number specifying which counts to use
start (int): An integer specifying the sequence start position
end (int): An integer specifying the sequence end position
Returns:
freq_df (pd.DataFrame): A dataframe containing results.
"""
# Validate dataset_df
qc.validate_dataset(dataset_df)
# Compute counts
counts_df = profile_ct.main(dataset_df, bin=bin, start=start, end=end)
# Create columns for profile_freqs table
ct_cols = [c for c in counts_df.columns if qc.is_col_type(c,'ct_')]
# Record positions in new dataframe
mut_df = counts_df[['pos']].copy()
# Compute mutation rate across counts
max_ct = counts_df[ct_cols].max(axis=1)
sum_ct = counts_df[ct_cols].sum(axis=1)
mut = 1.0 - (max_ct/sum_ct)
mut_df['mut'] = mut
# Computation of error rate is optional
if err:
mut_err = np.sqrt(mut*(1.0-mut)/sum_ct)
mut_df['mut_err'] = mut_err
# Figure out which alphabet the cts dataframe specifies
alphabet = ''.join([c.split('_')[1] for c in ct_cols])
seqtype = qc.alphabet_to_seqtype_dict[alphabet]
wt_col = qc.seqtype_to_wtcolname_dict[seqtype]
# Compute WT base at each position
mut_df[wt_col] = 'X'
for col in ct_cols:
indices = (counts_df[col]==max_ct).values
mut_df.loc[indices,wt_col] = col.split('_')[1]
# Validate as counts dataframe
mut_df = qc.validate_profile_mut(mut_df,fix=True)
return mut_df
def test_profile_ct_totalcounts(self):
""" Test the ability of sortseq_tools.profile_ct to count frequencies based on total count values
"""
print '\nIn test_profile_ct_totalcounts...'
library_files = glob.glob(self.input_dir+'library_*.txt')
library_files += glob.glob(self.input_dir+'dataset_*.txt')
for file_name in library_files:
print '\t%s ='%file_name,
description = file_name.split('_')[-1].split('.')[0]
executable = lambda: profile_ct.main(io.load_dataset(file_name))
# If good, then profile_ct.main should produce a valid df
if '_good' in file_name:
try:
df = executable()
qc.validate_profile_ct(df)
out_file = self.output_dir+\
'profile_ct_total_%s.txt'%description
io.write(df,out_file)
io.load_profile_ct(out_file)
print 'good.'
except:
print 'bad (ERROR).'
raise
# If bad, then profile_ct.main should raise SortSeqError
elif '_bad' in file_name:
try:
self.assertRaises(SortSeqError,executable)
print 'badtype.'
except:
print 'good (ERROR).'
raise
# There are no other options
else:
raise SortSeqError('Unrecognized class of file_name.')
def test_profile_ct_seqslicing(self):
""" Test the ability of sortseq_tools.profile_ct to slice sequences properly, and to raise the correct errors
"""
print '\nIn test_profile_ct_seqslicing...'
library_files = glob.glob(self.input_dir+'library_*.txt')
library_files += glob.glob(self.input_dir+'dataset_*.txt')
for file_name in library_files:
print '\t%s ='%file_name,
description = file_name.split('_')[-1].split('.')[0]
executable_good1 =\
lambda: profile_ct.main(io.load_dataset(file_name),\
start=2,end=10)
executable_good2 =\
lambda: profile_ct.main(io.load_dataset(file_name),\
start=2)
executable_good3 =\
lambda: profile_ct.main(io.load_dataset(file_name),\
end=2)
executable_nopro =\
lambda: profile_ct.main(io.load_dataset(file_name),\
start=50,end=60)
executable_bad1 =\
lambda: profile_ct.main(io.load_dataset(file_name),\
start=-1)
executable_bad2 =\
lambda: profile_ct.main(io.load_dataset(file_name),\
end=100)
executable_bad3 =\
lambda: profile_ct.main(io.load_dataset(file_name),\
start=20,end=10)
# If good, then sequences will be valid
if 'good' in file_name:
try:
df = executable_good1()
io.write(df,self.output_dir+\
'profile_ct_splice2-10_%s.txt'%description)
executable_good2()
executable_good3()
self.assertRaises(SortSeqError,executable_bad1)
self.assertRaises(SortSeqError,executable_bad2)
self.assertRaises(SortSeqError,executable_bad3)
if '_pro' in file_name:
self.assertRaises(SortSeqError,executable_nopro)
else:
df = executable_nopro()
print 'ok.'
except:
print 'ok (ERROR).'
raise
# If bad, then profile_ct.main should raise SortSeqError
elif '_bad' in file_name:
try:
self.assertRaises(SortSeqError,executable_good1)
self.assertRaises(SortSeqError,executable_good2)
self.assertRaises(SortSeqError,executable_good3)
self.assertRaises(SortSeqError,executable_nopro)
self.assertRaises(SortSeqError,executable_bad1)
self.assertRaises(SortSeqError,executable_bad2)
self.assertRaises(SortSeqError,executable_bad3)
print 'ok.'
except:
print 'not ok (ERROR).'
raise
# There are no other options
else:
raise SortSeqError('Unrecognized class of file_name.')
def main(dataset_df, err=False, method='naive',\
pseudocount=1.0, start=0, end=None):
"""
Computes the mutual information (in bits), at each position, between the character and the bin number.
Arguments:
dataset_df (pd.DataFrame): A dataframe containing a valid dataset.
start (int): An integer specifying the sequence start position
end (int): An integer specifying the sequence end position
method (str): Which method to use to estimate mutual information
Returns:
info_df (pd.DataFrame): A dataframe containing results.
"""
# Validate dataset_df
qc.validate_dataset(dataset_df)
# Get number of bins
bin_cols = [c for c in dataset_df.columns if qc.is_col_type(c,'ct_')]
if not len(bin_cols) >= 2:
raise SortSeqError('Information profile requires at least 2 bins.')
bins = [int(c.split('_')[1]) for c in bin_cols]
num_bins = len(bins)
# Get number of characters
seq_cols = [c for c in dataset_df.columns if qc.is_col_type(c,'seqs')]
if not len(seq_cols)==1:
raise SortSeqError('Must be only one seq column.')
seq_col = seq_cols[0]
seqtype = qc.colname_to_seqtype_dict[seq_col]
alphabet = qc.seqtype_to_alphabet_dict[seqtype]
ct_cols = ['ct_'+a for a in alphabet]
num_chars = len(alphabet)
# Get sequence length and check start, end numbers
num_pos = len(dataset_df[seq_col][0])
if not (0 <= start < num_pos):
raise SortSeqError('Invalid start==%d, num_pos==%d'%(start,num_pos))
if end is None:
end = num_pos
elif (end > num_pos):
raise SortSeqError('Invalid end==%d, num_pos==%d'%(end,num_pos))
elif end <= start:
raise SortSeqError('Invalid: start==%d >= end==%d'%(start,end))
# Record positions in new dataframe
counts_df = profile_ct.main(dataset_df)
info_df = counts_df.loc[start:(end-1),['pos']].copy() # rows from start:end
info_df['info'] = 0.0
if err:
info_df['info_err'] = 0.0
# Fill in 3D array of counts
ct_3d_array = np.zeros([end-start, num_chars, num_bins])
for i, bin_num in enumerate(bins):
# Compute counts
counts_df = profile_ct.main(dataset_df, bin=bin_num)
# Fill in counts table
ct_3d_array[:,:,i] = counts_df.loc[start:(end-1),ct_cols].astype(float)
# Compute mutual information for each position
for i in range(end-start): # i only from start:end
# Get 2D counts
nxy = ct_3d_array[i,:,:]
assert len(nxy.shape) == 2
# Compute mutual informaiton
if err:
mi, mi_err = info.estimate_mutualinfo(nxy,err=True,\
method=method,pseudocount=pseudocount)
info_df.loc[i+start,'info'] = mi
info_df.loc[i+start,'info_err'] = mi_err
else:
mi = info.estimate_mutualinfo(nxy,err=False,\
method=method,pseudocount=pseudocount)
info_df.loc[i+start,'info'] = mi
# Validate info dataframe
info_df = qc.validate_profile_info(info_df,fix=True)
return info_df
