lines= [x.decode() for x in lines]
refseq= lines[1].strip()
kmer_dict= fasta_get_freq(refseq,start= 0,end= 0,step= 1,ksize=ksize,bases= bases)
ref_kmer_prop =kmer_freq_balance(kmer_dict,mutations,fasta_len= len(refseq))
return ref_kmer_prop
ksize= 3
#bases= 'ATCG'
bases= 'ACGT'
mutations= get_mutations(bases= bases,ksize= ksize)
kmers, kmer_idx= kmer_comp_index(mutations)
mut_lib= kmer_mut_index(mutations)
from tools.mcounter_tools import read_args
p_value= 1e-5
test_m= 'chi2'
individually= False
exclude= False
frequency_range= [0,1]
extract= 'pval'
Nbins= 100
tag_ref= '_ss'
def vcf_muts_matrix_v1(refseq,summary,start= 0,end= 0,ksize= 3,bases='ATCG', collapse= True):
'''
Return matrix of mutation contexts by SNP in genotype array
Each mutation is mapped to list of possible mutations as a binary vector.
- v1 determines if alternative allele = reference allele in fasta.
if so, allele is switched, position idx is flagged.
'''
mutations= get_mutations(bases= bases,ksize= ksize)
kmers, kmer_idx= kmer_comp_index(mutations)
mut_lib= kmer_mut_index(mutations)
if end == 0:
end= max(summary.POS)
k5= int(ksize/2)
k3= ksize - k5
pos_mut= []
flag_reverse= []
flag_remove= []
for x in range(summary.shape[0]):
pos= int(summary.POS[x]) - 1
if pos >= start and pos <= end:
kmer= refseq[pos-k5: pos + k3]
if 'N' in kmer:
flag_remove.append(x)
continue
mut= kmer + summary.ALT[x]
if kmer[1] == summary.ALT[x]:
flag_reverse.append(x)
mut= kmer+summary.REF[x]
if len(mut) != 4:
print(kmer)
print(summary.REF[x],summary.ALT[x])
print(x,pos)
print(len(refseq),summary.shape[0])
if collapse:
mut_array=np.zeros(len(kmer_idx))
pos_mut.append(mut_array)
continue
else:
mut_array=np.zeros(len(mutations))
pos_mut.append(mut_array)
continue
if collapse:
mut_index= kmers[mut]
mut_array=np.zeros(len(kmer_idx))
else:
mut_index= get_by_path(mut_lib, list(mut))
mut_array=np.zeros(len(mutations))
mut_array[mut_index]= 1
pos_mut.append(mut_array)
pos_mut= np.array(pos_mut).T
return pos_mut, flag_reverse, flag_remove
def MC_sample_matrix_stdlone(sim,
out_db='out_db.txt',
min_size=80,
samp=[5, 20, 10],
stepup="increment",
diffs=False,
frequency_range=[0, 1],
indfile='ind_assignments.txt',
outemp='ind_assignments{}.txt',
chrom_idx=0,
prop_gen_used=1,
sim_dir='mutation_counter/data/sims/',
segregating=False,
scale_genSize=False,
outlog='indy.log',
row=24,
col=4,
single=False,
exclude=False,
print_summ=False,
sample_sim=0,
collapsed=True,
bases='ACGT',
ksize=3,
ploidy=2,
freq_extract=False,
sim_del='C',
tag_sim='_ss',
genome_size=1,
haps_extract=False,
return_private=True):
'''
'''
ti = time.time()
## chromosome
chrom = sim.split('.')[chrom_idx].split(sim_del)[-1].strip('chr')
pop_dict, inds = get_pop_dict(sim,
dir_sim=sim_dir,
indfile=indfile,
haps_extract=haps_extract,
return_inds=True)
if haps_extract:
inds = list(inds) + list(inds)
inds = np.array(inds)
print(len(inds))
print(inds[:10])
total_inds = sum([len(x) for x in pop_dict.values()])
if exclude:
files = read_exclude()
else:
files = {}
data_kmer = {}
tags = []
sim_extend = []
chroms = []
### read vcf
t0 = time.time()
Window, mut_matrix, scale = VCF_read_filter(sim,
sim_dir=sim_dir,
chrom=chrom,
haps_extract=haps_extract,
scale_genSize=scale_genSize,
collapsed=collapsed,
min_size=min_size,
samp=samp,
stepup=stepup,
outemp=outemp,
indfile=indfile,
diffs=diffs,
bases=bases,
ksize=ksize,
ploidy=ploidy)
#pop_dict= ()
#total_inds= sum([len(x) for x in pop_dict.values()])
t1 = time.time()
read_time = t1 - t0
if not len(Window) or Window.shape[0] < total_inds:
return ''
## counts for no tag sim:
s0 = time.time()
pop_summary, PA_dict = count_popKmers(Window,
mut_matrix,
pop_dict,
single=single,
prop_gen_used=prop_gen_used,
frequency_range=frequency_range,
row=row,
col=col,
segregating=segregating,
scale=scale,
return_private=return_private)
t2 = time.time()
print('time elapsed ref: {} m'.format((t2 - t1) / 60))
if return_private:
pop_summary, dummy = count_popKmers(Window,
mut_matrix,
pop_dict,
single=single,
prop_gen_used=prop_gen_used,
frequency_range=frequency_range,
row=row,
col=col,
segregating=segregating,
scale=scale,
PA=PA_dict)
data_kmer[sim] = pop_summary
t3 = time.time()
print('time elapsed ref PA: {} m'.format((t3 - t2) / 60))
new_wind = []
for pop in data_kmer[sim]['counts'].keys():
pop_counts = pop_summary['array'][pop_dict[pop], :]
pop_counts = np.array(pop_counts, dtype=int)
Service = np.zeros((pop_counts.shape[0], pop_counts.shape[1] + 3),
dtype=int)
Service[:, 3:] = pop_counts
Service = np.array(Service, dtype=str)
Service[:, 2] = inds[pop_dict[pop]]
Service[:, 1] = pop
Service[:, 0] = sim
new_wind.append(Service)
new_wind = np.concatenate(tuple(new_wind), axis=0)
### mutation labels
mutations = get_mutations(bases=bases, ksize=ksize)
kmers, kmer_idx = kmer_comp_index(mutations)
mut_lib = kmer_mut_index(mutations)
if collapsed:
labels = [kmer_idx[x][0] for x in sorted(kmer_idx.keys())]
else:
labels = ['_'.join(x) for x in mutations]
# write:
db_file = out_db.format(sim)
db_neigh = db_file.split('/')[:-1]
db_name = db_file.split('/')[-1]
db_neigh = os.listdir('/'.join(db_neigh))
header = ['SIM', 'POP', 'N'] + labels
header = '\t'.join(header) + '\n'
if db_name not in db_neigh:
with open(db_file, 'w') as fp:
fp.write(header)
with open(db_file, 'w') as fp:
fp.write(header)
fp.write('\n'.join(['\t'.join(x) for x in new_wind]))
t1 = time.time()
count_time = t1 - t0
return ''
def MC_sample_matrix_stdlone(sim,out_db= 'out_db.txt',min_size= 80, samp= [5,20,10], stepup= "increment", diffs= False, frequency_range= [0,1],indfile= 'ind_assignments.txt',
outemp= 'ind_assignments{}.txt',chrom_idx= 0, prop_gen_used= 1,
sim_dir= 'mutation_counter/data/sims/', segregating= False, scale_genSize= False,
outlog= 'indy.log', row= 24,col= 4, single= True, exclude= False, print_summ= False, sample_sim= 0,
collapsed= True,bases= 'ACGT',ksize= 3,ploidy= 2, freq_extract= False, sim_del= 'C', tag_sim= '_ss',
genome_size= 1,haps_extract= False, return_private= True):
'''
'''
### prepare sim_specific db
###
### mutation labels
mutations= get_mutations(bases= bases,ksize= ksize)
kmers, kmer_idx= kmer_comp_index(mutations)
mut_lib= kmer_mut_index(mutations)
if collapsed:
labels= [kmer_idx[x][0] for x in sorted(kmer_idx.keys())]
else:
labels= ['_'.join(x) for x in mutations]
# write:
out_db= out_db.format(sim)
db_dir= out_db.split('/')
db_name= db_dir[-1]
db_dir= '/'.join(db_dir[:-1])+'/'
db_neigh= os.listdir(db_dir)
header= ['SIM','POP','N'] + labels
header= '\t'.join(header) + '\n'
if db_name not in db_neigh:
with open(out_db,'w') as fp:
fp.write(header)
###
###
ti= time.time()
## chromosome
chrom= sim.split('.')[chrom_idx].split(sim_del)[-1].strip('chr')
pop_dict, inds= get_pop_dict(sim,dir_sim= sim_dir,indfile= indfile,haps_extract= haps_extract,
return_inds= True)
if haps_extract:
inds= list(inds) + list(inds)
inds= np.array(inds)
print(len(inds))
print(inds[:10])
total_inds= sum([len(x) for x in pop_dict.values()])
if exclude:
files= read_exclude()
else:
files= {}
data_kmer= {}
tags= []
sim_extend= []
chroms= []
### read vcf
t0= time.time()
Window, mut_matrix, scale= VCF_read_filter(sim, sim_dir= sim_dir,chrom= chrom,haps_extract= haps_extract, scale_genSize= scale_genSize,
collapsed= collapsed,min_size= min_size, samp= samp, stepup= stepup, outemp= outemp,
indfile= indfile,diffs= diffs,bases= bases, ksize= ksize, ploidy= ploidy)
mut_idx= np.argmax(mut_matrix,axis= 0)
tag_list, tag_dict, pop_dict= ind_assignment_scatter_v1(sim,dir_sim= sim_dir,
min_size= min_size, samp= samp, stepup= stepup, outemp= outemp,indfile= indfile,
inds= inds, pop_dict= pop_dict,pop_sub= True)
t1= time.time()
read_time= t1- t0
if not len(Window) or Window.shape[0] < total_inds:
return ''
## counts for no tag sim:
s0= time.time()
PA_dict= parse_PA(Window, pop_dict,frequency_range= frequency_range)
t2= time.time()
print('time elapsed ref: {} m'.format((t2 - t1)/60))
new_wind= {}
for pop in pop_dict.keys():
klist= list(pop_dict[pop])
private= list(PA_dict[pop])
pop_wind= Window[klist,:]
pop_wind= pop_wind[:,private]
##
new_wind[pop]= {
'array': pop_wind,
'mut': [mut_idx[x] for x in private]
}
##
#
for pop in pop_dict.keys():
dict_pop= {pop: list(range(len(pop_dict[pop])))}
pop_summary, dummy= count_popKmers(new_wind[pop]['array'], mut_matrix, new_wind[pop]['mut'], dict_pop,
row=row,col=col)
nline= [sim,pop,len(pop_dict[pop])]
ncounts= pop_summary['counts'][pop]
ncounts= ncounts.reshape(1,np.prod(ncounts.shape))[0]
nline= nline + list(ncounts)
nline= [str(x) for x in nline]
with open(out_db,'a') as fp:
fp.write('\t'.join(nline) + '\n')
t3= time.time()
print('time elapsed ref PA: {} m'.format((t3 - t2)/60))
if len(tag_list):
###
print('len tag list: {}'.format(len(tag_list)))
###
for idx in range(len(tag_list)):
tag= tag_list[idx]
pop_dict= tag_dict[tag]
pop_ori= list(pop_dict.keys())[0]
if tag_sim in pop_ori:
pop_ori= pop_ori[len(tag_sim):].split('.')[0]
pop_summary, dummy= count_popKmers(new_wind[pop_ori]['array'], mut_matrix, new_wind[pop_ori]['mut'], pop_dict,
row=row,col=col,segregating= True,single= True)
for pop in pop_summary['counts'].keys():
nline= [sim,pop,len(pop_dict[pop])]
ncounts= pop_summary['counts'][pop]
ncounts= ncounts.reshape(1,np.prod(ncounts.shape))[0]
nline= nline + list(ncounts)
nline= [str(x) for x in nline]
with open(out_db,'a') as fp:
fp.write('\t'.join(nline) + '\n')
return ''
def mcounter_deploy_v2(data,
p_value=1e-5,
test_m='fisher',
chi_total=False,
ksize=3,
bases='ACGT',
frequency_range=[0, 1],
data_freqs={},
extract='pval',
row=64,
col=3,
tag_ref='_ss',
collapsed=False,
bsep='C',
muted_dir='',
sims_dir='mutation_counter/data/sims/',
fasta_var=True):
'''
Parse data dictionary.
data: {sim: {counts:{pop:g}, Nvars:{pop:g}, sizes:{pop:g}}}
i: use sim and pop IDs to create dictionary connecting original populations to
subset populations created using ind_assignment_scatter_v1.
ii: for each pair of reference/subset populations, launch heatmapv2. return grid pvals or proportions,
and proportion of mutations in subset population. allows for fisher or chi2 test for pval.
- v2: compares sub pops to ref full pops other than its own; gets store of differences among refs.
'''
mutations = get_mutations(bases=bases, ksize=ksize)
avail = list(data.keys())
ref_idx = [int(tag_ref in avail[x]) for x in range(len(avail))]
categ = {
z: [x for x in range(len(avail)) if ref_idx[x] == z]
for z in [0, 1]
}
fasta_ref_dict = {}
pop_asso = {avail[x]: recursively_default_dict() for x in categ[0]}
ref_batch_dict = recursively_default_dict()
reference_freqs = {}
fasta_count_array = []
for idx in categ[0]:
ref = avail[idx]
batch = bsep.join(ref.split(bsep)[:-1])
ref_batch_dict[batch][ref] = ''
ref_dir = sims_dir + ref + '/'
fasta_kmer_prop = get_fasta_prop(ref,
ref_dir,
mutations,
ksize=ksize,
bases=bases,
collapsed=collapsed)
fasta_ref_dict[ref] = fasta_kmer_prop
fasta_count_array.extend(fasta_kmer_prop)
### get fasta var
fasta_count_array = np.array(fasta_count_array)
print(fasta_count_array.shape)
print('fasta array shape: {}'.format(fasta_count_array.shape))
fasta_var = set_SSD(fasta_count_array, fasta_count_array, same=True)
fasta_mtype_var = np.std(fasta_count_array, axis=0)
fasta_var_dict = {'dist': fasta_var, 'type_sd': fasta_mtype_var}
###
ref_batch_dict = {z: [] for z, g in ref_batch_dict.items()}
if data_freqs:
reference_freqs = {x: {} for x in pop_asso.keys()}
for av in categ[1]:
dat = [x for x in data[avail[av]]['counts'].keys() if tag_ref in x]
ref_sim = avail[av].split(tag_ref)[0]
ref_pop = [x.split('.')[0].strip(tag_ref) for x in dat]
for p in range(len(dat)):
pop_asso[ref_sim][ref_pop[p]][avail[av]] = dat[p]
d = 0
count_data = recursively_default_dict()
for ref in pop_asso.keys():
batch = bsep.join(ref.split(bsep)[:-1])
ref_count = fasta_ref_dict[ref]
for pop in pop_asso[ref].keys():
ref_pop_counts = data[ref]['counts'][pop]
ref_pop_counts = ref_pop_counts / np.sum(ref_pop_counts)
pop_shape = ref_pop_counts.shape
ref_pop_counts = ref_pop_counts.reshape(1, np.prod(pop_shape))
ref_pop_counts = (ref_pop_counts - ref_count / 3) / (1 - ref_count)
ref_pop_counts = ref_pop_counts.reshape(*pop_shape)
ref_batch_dict[batch].append((pop, ref, ref_pop_counts))
if data_freqs:
reference_freqs[ref][pop] = data_freqs[ref][pop]
for sub in pop_asso[ref][pop].keys():
ref_pair = [(ref, pop), (sub, pop_asso[ref][pop][sub])]
fasta_dict_local = {ref: ref_count, sub: fasta_ref_dict[ref]}
count_data[d] = run_stats(ref,
ref_pair,
data,
data_freqs={},
fasta_dict=fasta_dict_local,
bsep=bsep,
row=row,
col=col,
test_m=test_m,
chi_total=chi_total)
count_data[d]['pop'] = pop
count_data[d]['other'] = []
#for ref2 in pop_asso.keys():
for pop2 in pop_asso[ref].keys():
if pop == pop2:
continue
if bsep.join(ref.split(bsep)[:-1]) != batch:
continue
##
pop_dict = {ref: pop2, sub: pop_asso[ref][pop][sub]}
fasta_dict_local = {ref: ref_count, sub: ref_count}
ref_pair = [(ref, pop2), (sub, pop_asso[ref][pop][sub])]
pair_stats = run_stats(ref,
ref_pair,
data,
data_freqs={},
fasta_dict=fasta_dict_local,
bsep=bsep,
test_m=test_m,
row=row,
col=col,
chi_total=chi_total)
count_data[d]['other'].append(pair_stats['diffs'])
d += 1
####
#### Reference counts, comparison by batch.
if len(fasta_var):
return pop_asso, count_data, ref_batch_dict, reference_freqs, fasta_var_dict
else:
return pop_asso, count_data, ref_batch_dict, reference_freqs