def collapse_mat(count_props_array, muts):
'''
collapse counts_props_array.
'''
muts_tuple = [x.split('_') for x in muts]
mutations = [tuple(x) for x in muts_tuple]
kmers, kmer_idx = kmer_comp_index(mutations)
muts_idx = [kmers[''.join(x)] for x in mutations]
ncount_array = np.zeros((count_props_array.shape[0], len(kmer_idx)))
for idx in range(len(muts_idx)):
nidx = muts_idx[idx]
ncount_array[:, nidx] += count_props_array[:, idx]
count_props_array = ncount_array
return count_props_array, kmers
refseq= lines[1].strip()
kmer_dict= fasta_get_freq(refseq,start= 0,end= 0,step= 1,ksize=ksize,bases= bases)
ref_kmer_prop =kmer_freq_balance(kmer_dict,mutations,fasta_len= len(refseq))
return ref_kmer_prop
ksize= 3
#bases= 'ATCG'
bases= 'ACGT'
mutations= get_mutations(bases= bases,ksize= ksize)
kmers, kmer_idx= kmer_comp_index(mutations)
mut_lib= kmer_mut_index(mutations)
from tools.mcounter_tools import read_args
p_value= 1e-5
test_m= 'chi2'
individually= False
exclude= False
frequency_range= [0,1]
extract= 'pval'
Nbins= 100
tag_ref= '_ss'
def vcf_muts_matrix_v1(refseq,summary,start= 0,end= 0,ksize= 3,bases='ATCG', collapse= True):
'''
Return matrix of mutation contexts by SNP in genotype array
Each mutation is mapped to list of possible mutations as a binary vector.
- v1 determines if alternative allele = reference allele in fasta.
if so, allele is switched, position idx is flagged.
'''
mutations= get_mutations(bases= bases,ksize= ksize)
kmers, kmer_idx= kmer_comp_index(mutations)
mut_lib= kmer_mut_index(mutations)
if end == 0:
end= max(summary.POS)
k5= int(ksize/2)
k3= ksize - k5
pos_mut= []
flag_reverse= []
flag_remove= []
for x in range(summary.shape[0]):
pos= int(summary.POS[x]) - 1
if pos >= start and pos <= end:
kmer= refseq[pos-k5: pos + k3]
if 'N' in kmer:
flag_remove.append(x)
continue
mut= kmer + summary.ALT[x]
if kmer[1] == summary.ALT[x]:
flag_reverse.append(x)
mut= kmer+summary.REF[x]
if len(mut) != 4:
print(kmer)
print(summary.REF[x],summary.ALT[x])
print(x,pos)
print(len(refseq),summary.shape[0])
if collapse:
mut_array=np.zeros(len(kmer_idx))
pos_mut.append(mut_array)
continue
else:
mut_array=np.zeros(len(mutations))
pos_mut.append(mut_array)
continue
if collapse:
mut_index= kmers[mut]
mut_array=np.zeros(len(kmer_idx))
else:
mut_index= get_by_path(mut_lib, list(mut))
mut_array=np.zeros(len(mutations))
mut_array[mut_index]= 1
pos_mut.append(mut_array)
pos_mut= np.array(pos_mut).T
return pos_mut, flag_reverse, flag_remove
def MC_sample_matrix_stdlone(sim,out_db= 'out_db.txt',min_size= 80, samp= [5,20,10], stepup= "increment", diffs= False, frequency_range= [0,1],indfile= 'ind_assignments.txt',
outemp= 'ind_assignments{}.txt',chrom_idx= 0, prop_gen_used= 1,
sim_dir= 'mutation_counter/data/sims/', segregating= False, scale_genSize= False,
outlog= 'indy.log', row= 24,col= 4, single= True, exclude= False, print_summ= False, sample_sim= 0,
collapsed= True,bases= 'ACGT',ksize= 3,ploidy= 2, freq_extract= False, sim_del= 'C', tag_sim= '_ss',
genome_size= 1,haps_extract= False, return_private= True):
'''
'''
### prepare sim_specific db
###
### mutation labels
mutations= get_mutations(bases= bases,ksize= ksize)
kmers, kmer_idx= kmer_comp_index(mutations)
mut_lib= kmer_mut_index(mutations)
if collapsed:
labels= [kmer_idx[x][0] for x in sorted(kmer_idx.keys())]
else:
labels= ['_'.join(x) for x in mutations]
# write:
out_db= out_db.format(sim)
db_dir= out_db.split('/')
db_name= db_dir[-1]
db_dir= '/'.join(db_dir[:-1])+'/'
db_neigh= os.listdir(db_dir)
header= ['SIM','POP','N'] + labels
header= '\t'.join(header) + '\n'
if db_name not in db_neigh:
with open(out_db,'w') as fp:
fp.write(header)
###
###
ti= time.time()
## chromosome
chrom= sim.split('.')[chrom_idx].split(sim_del)[-1].strip('chr')
pop_dict, inds= get_pop_dict(sim,dir_sim= sim_dir,indfile= indfile,haps_extract= haps_extract,
return_inds= True)
if haps_extract:
inds= list(inds) + list(inds)
inds= np.array(inds)
print(len(inds))
print(inds[:10])
total_inds= sum([len(x) for x in pop_dict.values()])
if exclude:
files= read_exclude()
else:
files= {}
data_kmer= {}
tags= []
sim_extend= []
chroms= []
### read vcf
t0= time.time()
Window, mut_matrix, scale= VCF_read_filter(sim, sim_dir= sim_dir,chrom= chrom,haps_extract= haps_extract, scale_genSize= scale_genSize,
collapsed= collapsed,min_size= min_size, samp= samp, stepup= stepup, outemp= outemp,
indfile= indfile,diffs= diffs,bases= bases, ksize= ksize, ploidy= ploidy)
mut_idx= np.argmax(mut_matrix,axis= 0)
tag_list, tag_dict, pop_dict= ind_assignment_scatter_v1(sim,dir_sim= sim_dir,
min_size= min_size, samp= samp, stepup= stepup, outemp= outemp,indfile= indfile,
inds= inds, pop_dict= pop_dict,pop_sub= True)
t1= time.time()
read_time= t1- t0
if not len(Window) or Window.shape[0] < total_inds:
return ''
## counts for no tag sim:
s0= time.time()
PA_dict= parse_PA(Window, pop_dict,frequency_range= frequency_range)
t2= time.time()
print('time elapsed ref: {} m'.format((t2 - t1)/60))
new_wind= {}
for pop in pop_dict.keys():
klist= list(pop_dict[pop])
private= list(PA_dict[pop])
pop_wind= Window[klist,:]
pop_wind= pop_wind[:,private]
##
new_wind[pop]= {
'array': pop_wind,
'mut': [mut_idx[x] for x in private]
}
##
#
for pop in pop_dict.keys():
dict_pop= {pop: list(range(len(pop_dict[pop])))}
pop_summary, dummy= count_popKmers(new_wind[pop]['array'], mut_matrix, new_wind[pop]['mut'], dict_pop,
row=row,col=col)
nline= [sim,pop,len(pop_dict[pop])]
ncounts= pop_summary['counts'][pop]
ncounts= ncounts.reshape(1,np.prod(ncounts.shape))[0]
nline= nline + list(ncounts)
nline= [str(x) for x in nline]
with open(out_db,'a') as fp:
fp.write('\t'.join(nline) + '\n')
t3= time.time()
print('time elapsed ref PA: {} m'.format((t3 - t2)/60))
if len(tag_list):
###
print('len tag list: {}'.format(len(tag_list)))
###
for idx in range(len(tag_list)):
tag= tag_list[idx]
pop_dict= tag_dict[tag]
pop_ori= list(pop_dict.keys())[0]
if tag_sim in pop_ori:
pop_ori= pop_ori[len(tag_sim):].split('.')[0]
pop_summary, dummy= count_popKmers(new_wind[pop_ori]['array'], mut_matrix, new_wind[pop_ori]['mut'], pop_dict,
row=row,col=col,segregating= True,single= True)
for pop in pop_summary['counts'].keys():
nline= [sim,pop,len(pop_dict[pop])]
ncounts= pop_summary['counts'][pop]
ncounts= ncounts.reshape(1,np.prod(ncounts.shape))[0]
nline= nline + list(ncounts)
nline= [str(x) for x in nline]
with open(out_db,'a') as fp:
fp.write('\t'.join(nline) + '\n')
return ''
def MC_sample_matrix_stdlone(sim,
out_db='out_db.txt',
min_size=80,
samp=[5, 20, 10],
stepup="increment",
diffs=False,
frequency_range=[0, 1],
indfile='ind_assignments.txt',
outemp='ind_assignments{}.txt',
chrom_idx=0,
prop_gen_used=1,
sim_dir='mutation_counter/data/sims/',
segregating=False,
scale_genSize=False,
outlog='indy.log',
row=24,
col=4,
single=False,
exclude=False,
print_summ=False,
sample_sim=0,
collapsed=True,
bases='ACGT',
ksize=3,
ploidy=2,
freq_extract=False,
sim_del='C',
tag_sim='_ss',
genome_size=1,
haps_extract=False,
return_private=True):
'''
'''
ti = time.time()
## chromosome
chrom = sim.split('.')[chrom_idx].split(sim_del)[-1].strip('chr')
pop_dict, inds = get_pop_dict(sim,
dir_sim=sim_dir,
indfile=indfile,
haps_extract=haps_extract,
return_inds=True)
if haps_extract:
inds = list(inds) + list(inds)
inds = np.array(inds)
print(len(inds))
print(inds[:10])
total_inds = sum([len(x) for x in pop_dict.values()])
if exclude:
files = read_exclude()
else:
files = {}
data_kmer = {}
tags = []
sim_extend = []
chroms = []
### read vcf
t0 = time.time()
Window, mut_matrix, scale = VCF_read_filter(sim,
sim_dir=sim_dir,
chrom=chrom,
haps_extract=haps_extract,
scale_genSize=scale_genSize,
collapsed=collapsed,
min_size=min_size,
samp=samp,
stepup=stepup,
outemp=outemp,
indfile=indfile,
diffs=diffs,
bases=bases,
ksize=ksize,
ploidy=ploidy)
#pop_dict= ()
#total_inds= sum([len(x) for x in pop_dict.values()])
t1 = time.time()
read_time = t1 - t0
if not len(Window) or Window.shape[0] < total_inds:
return ''
## counts for no tag sim:
s0 = time.time()
pop_summary, PA_dict = count_popKmers(Window,
mut_matrix,
pop_dict,
single=single,
prop_gen_used=prop_gen_used,
frequency_range=frequency_range,
row=row,
col=col,
segregating=segregating,
scale=scale,
return_private=return_private)
t2 = time.time()
print('time elapsed ref: {} m'.format((t2 - t1) / 60))
if return_private:
pop_summary, dummy = count_popKmers(Window,
mut_matrix,
pop_dict,
single=single,
prop_gen_used=prop_gen_used,
frequency_range=frequency_range,
row=row,
col=col,
segregating=segregating,
scale=scale,
PA=PA_dict)
data_kmer[sim] = pop_summary
t3 = time.time()
print('time elapsed ref PA: {} m'.format((t3 - t2) / 60))
new_wind = []
for pop in data_kmer[sim]['counts'].keys():
pop_counts = pop_summary['array'][pop_dict[pop], :]
pop_counts = np.array(pop_counts, dtype=int)
Service = np.zeros((pop_counts.shape[0], pop_counts.shape[1] + 3),
dtype=int)
Service[:, 3:] = pop_counts
Service = np.array(Service, dtype=str)
Service[:, 2] = inds[pop_dict[pop]]
Service[:, 1] = pop
Service[:, 0] = sim
new_wind.append(Service)
new_wind = np.concatenate(tuple(new_wind), axis=0)
### mutation labels
mutations = get_mutations(bases=bases, ksize=ksize)
kmers, kmer_idx = kmer_comp_index(mutations)
mut_lib = kmer_mut_index(mutations)
if collapsed:
labels = [kmer_idx[x][0] for x in sorted(kmer_idx.keys())]
else:
labels = ['_'.join(x) for x in mutations]
# write:
db_file = out_db.format(sim)
db_neigh = db_file.split('/')[:-1]
db_name = db_file.split('/')[-1]
db_neigh = os.listdir('/'.join(db_neigh))
header = ['SIM', 'POP', 'N'] + labels
header = '\t'.join(header) + '\n'
if db_name not in db_neigh:
with open(db_file, 'w') as fp:
fp.write(header)
with open(db_file, 'w') as fp:
fp.write(header)
fp.write('\n'.join(['\t'.join(x) for x in new_wind]))
t1 = time.time()
count_time = t1 - t0
return ''
