def build_representative_cluster(clustering_path, threads, input_prefix):
""" build representative cluster """
start = time.time()
cluster_file = ''.join([clustering_path, input_prefix, '_cluster.output'])
representative_outputfile = ''.join(
[clustering_path, input_prefix, '_representative', '.faa'])
subproblem_seqs_path = '%ssubproblem_cluster_seqs/' % clustering_path
subproblem_merged_faa = ''.join([clustering_path, input_prefix, '.faa'])
subproblem_faa_dict = read_fasta(subproblem_merged_faa)
with open(cluster_file, 'rb') as cluster_input:
subproblem_geneCluster_dt = defaultdict(list)
cluster_input_lines = [iline for iline in cluster_input]
subproblem_geneCluster_dt = {}
subproblem_run_number = input_prefix.split('subproblem_')[1]
for gid, iline in enumerate(cluster_input_lines): #cluster_input
## use time to avoid clusterID conflict
clusterID = "GCs%s_%07d%s" % (subproblem_run_number, gid,
time.strftime('%M%S', time.gmtime()))
gene_ids = iline.rstrip().split('\t')
subproblem_geneCluster_dt[clusterID] = gene_ids
## representative_seq
representative_seq = subproblem_faa_dict[gene_ids[0]]
## write in representative strain
with open(representative_outputfile, 'a') as representative_output:
write_in_fa(representative_output, clusterID,
representative_seq)
## write subproblem_geneCluster_dt
write_pickle(''.join([clustering_path, input_prefix, '_dicts.cpk']),
subproblem_geneCluster_dt)
print 'build representative clusters for', input_prefix, ': ', times(
start), '\n'
def create_RNACluster_fa(path, folders_dict):
"""
input: '.fna', '_RNA_nuc_dict.cpk', 'allclusters.cpk'
output: '.aln', 'tree.json', etc
"""
RNA_path = folders_dict['RNA_path']
RNA_dict = load_pickle('%s%s' % (RNA_path, 'all_RNA_seq.cpk'))
## load RNA cluster cpk file
diamond_RNACluster_dt = load_pickle(RNA_path + 'allclusters.cpk')
## load RNAID_to_RNASeqID RNASeqID cpk file
RNAID_to_RNASeqID_dict = load_pickle(path + 'RNAID_to_SeqID.cpk')
## create cluster-RNAs fasta files (By default: put RNAs in geneCluster folder)
fasta_path = path + 'geneCluster/'
## diamond_RNACluster_dt: {clusterID:[ count_strains,[memb1,...],count_RNAs }
for clusterID, RNA in diamond_RNACluster_dt.iteritems():
## RNACluster file name
RNA_cluster_nu_filename = "%s%s" % (clusterID, '.fna')
RNA_cluster_nu_write = open(fasta_path + RNA_cluster_nu_filename, 'wb')
## write nucleotide/amino_acid sequences into RNACluster files
for RNA_memb in RNA[1]:
## RNA_name format: strain_1|locusTag
strain_name = RNA_memb.split('|')[0]
RNA_memb_seq = str(RNA_dict[strain_name][RNA_memb])
RNASeqID = RNAID_to_RNASeqID_dict[RNA_memb]
write_in_fa(RNA_cluster_nu_write, RNASeqID, RNA_memb_seq)
RNA_cluster_nu_write.close()
return diamond_RNACluster_dt
def single_RNACluster_align_and_makeTree(fa_files_list, alignFile_path, simple_tree):
fasttree_name= 'fasttree' if check_dependency('fasttree') else 'FastTree'
for RNA_cluster_nu_filename in fa_files_list:
try:
# extract GC_RNA002 from path/GC_RNA002.aln
clusterID = RNA_cluster_nu_filename.split('/')[-1].split('.')[0]
geneDiversity_file = open(alignFile_path+'gene_diversity.txt', 'a')
if len( read_fasta(RNA_cluster_nu_filename) )==1: # nothing to do for singletons
## na.aln
RNA_cluster_nu_aln_filename= RNA_cluster_nu_filename.replace('.fna','_na.aln')
## RNA SeqID separator '|' is replaced by '-' for msa viewer compatibility
with open(RNA_cluster_nu_aln_filename,'wb') as write_file:
for SeqID, Sequence in read_fasta(RNA_cluster_nu_filename).iteritems():
write_in_fa(write_file, SeqID.replace('|','-'), Sequence)
geneDiversity_file.write('%s\t%s\n'%(clusterID,'0.0'))
else: # align and build tree
print RNA_cluster_nu_filename
myTree = mpm_tree(RNA_cluster_nu_filename)
myTree.align()
if simple_tree==False:
myTree.build(raxml=False,fasttree_program=fasttree_name,treetime_used=True)
myTree.ancestral(translate_tree=True)
myTree.refine(CDS=False)
else:
myTree.build(raxml=False,fasttree_program=fasttree_name,treetime_used=False)
myTree.diversity_statistics_nuc()
myTree.export(path=alignFile_path, RNA_specific=True)
RNA_diversity_values='{0:.3f}'.format(myTree.diversity_nuc)
geneDiversity_file.write('%s\t%s\n'%(clusterID,RNA_diversity_values))
print clusterID,RNA_diversity_values
except:
print("Aligning and tree building of RNA %s failed"%RNA_cluster_nu_filename)
def create_geneCluster_fa(path,folders_dict):
""" dict storing amino_acid Id/Seq from '.faa' files
input: '.faa', '_gene_nuc_dict.cpk', 'allclusters.cpk'
output:
"""
## make sure the geneCluster folder is empty
os.system('rm -rf %s'%(path+'geneCluster/'))
clustering_path= folders_dict['clustering_path']
geneCluster_dt= load_pickle(clustering_path+'allclusters.cpk')
protein_path= folders_dict['protein_path']
nucleotide_path= folders_dict['nucleotide_path']
geneID_to_geneSeqID_dict=load_pickle(path+'geneID_to_geneSeqID.cpk')
gene_aa_dict= load_pickle('%s%s'%(protein_path,'all_protein_seq.cpk'))
gene_na_dict= load_pickle('%s%s'%(nucleotide_path,'all_nucleotide_seq.cpk'))
## create cluster-genes fasta files
cluster_seqs_path=path+'geneCluster/'
os.system('mkdir '+cluster_seqs_path)
## write nuc/aa sequences for each cluster
for clusterID, gene in geneCluster_dt.iteritems():
## geneCluster file name
gene_cluster_nu_filename="%s%s"%(clusterID,'.fna')
gene_cluster_aa_filename="%s%s"%(clusterID,'.faa')
with open( cluster_seqs_path+gene_cluster_nu_filename, 'wb') as gene_cluster_nu_write, \
open( cluster_seqs_path+gene_cluster_aa_filename, 'wb') as gene_cluster_aa_write:
## write nucleotide/amino_acid sequences into geneCluster files
for gene_memb in gene[1]:
## gene_name format: strain_1|locusTag
strain_name= gene_memb.split('|')[0]
geneSeqID=geneID_to_geneSeqID_dict[gene_memb]
write_in_fa(gene_cluster_nu_write, geneSeqID, gene_na_dict[strain_name][gene_memb] )
write_in_fa(gene_cluster_aa_write, geneSeqID, gene_aa_dict[strain_name][gene_memb])
def create_geneCluster_fa(path,folders_dict):
""" dict storing amino_acid Id/Seq from '.faa' files
input: '.faa', '_gene_nuc_dict.cpk', 'allclusters.cpk'
output:
"""
## make sure the geneCluster folder is empty
os.system('rm -rf %s'%(path+'geneCluster/'))
clustering_path= folders_dict['clustering_path']
geneCluster_dt= load_pickle(clustering_path+'allclusters.cpk')
protein_path= folders_dict['protein_path']
nucleotide_path= folders_dict['nucleotide_path']
geneID_to_geneSeqID_dict=load_pickle(path+'geneID_to_geneSeqID.cpk')
gene_aa_dict= load_pickle('%s%s'%(protein_path,'all_protein_seq.cpk'))
gene_na_dict= load_pickle('%s%s'%(nucleotide_path,'all_nucleotide_seq.cpk'))
## create cluster-genes fasta files
cluster_seqs_path=path+'geneCluster/'
os.system('mkdir '+cluster_seqs_path)
## write nuc/aa sequences for each cluster
for clusterID, gene in geneCluster_dt.iteritems():
## geneCluster file name
gene_cluster_nu_filename="%s%s"%(clusterID,'.fna')
gene_cluster_aa_filename="%s%s"%(clusterID,'.faa')
with open( cluster_seqs_path+gene_cluster_nu_filename, 'wb') as gene_cluster_nu_write, \
open( cluster_seqs_path+gene_cluster_aa_filename, 'wb') as gene_cluster_aa_write:
## write nucleotide/amino_acid sequences into geneCluster files
for gene_memb in gene[1]:
## gene_name format: strain_1|locusTag
strain_name= gene_memb.split('|')[0]
geneSeqID=geneID_to_geneSeqID_dict[gene_memb]
write_in_fa(gene_cluster_nu_write, geneSeqID, gene_na_dict[strain_name][gene_memb] )
write_in_fa(gene_cluster_aa_write, geneSeqID, gene_aa_dict[strain_name][gene_memb])
def create_RNACluster_fa(path,folders_dict):
"""
input: '.fna', '_RNA_nuc_dict.cpk', 'allclusters.cpk'
output: '.aln', 'tree.json', etc
"""
RNA_path= folders_dict['RNA_path']
RNA_dict= load_pickle('%s%s'%(RNA_path,'all_RNA_seq.cpk'))
## load RNA cluster cpk file
diamond_RNACluster_dt=load_pickle(RNA_path+'allclusters.cpk')
## load RNAID_to_RNASeqID RNASeqID cpk file
RNAID_to_RNASeqID_dict=load_pickle(path+'RNAID_to_SeqID.cpk')
## create cluster-RNAs fasta files (By default: put RNAs in geneCluster folder)
fasta_path=path+'geneCluster/';
## diamond_RNACluster_dt: {clusterID:[ count_strains,[memb1,...],count_RNAs }
for clusterID, RNA in diamond_RNACluster_dt.iteritems():
## RNACluster file name
RNA_cluster_nu_filename="%s%s"%(clusterID,'.fna')
RNA_cluster_nu_write=open( fasta_path+RNA_cluster_nu_filename, 'wb')
## write nucleotide/amino_acid sequences into RNACluster files
for RNA_memb in RNA[1]:
## RNA_name format: strain_1|locusTag
strain_name= RNA_memb.split('|')[0]
RNA_memb_seq=str(RNA_dict[strain_name][RNA_memb])
RNASeqID=RNAID_to_RNASeqID_dict[RNA_memb]
write_in_fa(RNA_cluster_nu_write, RNASeqID, RNA_memb_seq )
RNA_cluster_nu_write.close()
return diamond_RNACluster_dt
def build_representative_cluster(clustering_path, threads, input_prefix):
""" build representative cluster """
start = time.time()
cluster_file= ''.join([clustering_path,input_prefix,'_cluster.output'])
representative_outputfile= ''.join([clustering_path,input_prefix,'_representative','.faa'])
subproblem_seqs_path= '%ssubproblem_cluster_seqs/'%clustering_path
subproblem_merged_faa= ''.join([clustering_path,input_prefix,'.faa'])
subproblem_faa_dict= read_fasta(subproblem_merged_faa)
with open(cluster_file, 'rb') as cluster_input:
subproblem_geneCluster_dt= defaultdict(list)
cluster_input_lines= [iline for iline in cluster_input]
subproblem_geneCluster_dt= {}
subproblem_run_number= input_prefix.split('subproblem_')[1]
for gid, iline in enumerate(cluster_input_lines):#cluster_input
## use time to avoid clusterID conflict
clusterID= "GCs%s_%07d%s"%(subproblem_run_number, gid, time.strftime('%M%S',time.gmtime()))
gene_ids= iline.rstrip().split('\t')
subproblem_geneCluster_dt[clusterID]= gene_ids
## representative_seq
representative_seq=subproblem_faa_dict[gene_ids[0]]
## write in representative strain
with open(representative_outputfile, 'a') as representative_output:
write_in_fa(representative_output, clusterID, representative_seq)
## write subproblem_geneCluster_dt
write_pickle(''.join([clustering_path,input_prefix,'_dicts.cpk']), subproblem_geneCluster_dt)
print 'build representative clusters for', input_prefix,': ', times(start), '\n'
def align_and_makeTree( fna_file_list, alignFile_path, simple_tree):
fasttree_name= 'fasttree' if check_dependency('fasttree') else 'FastTree'
for gene_cluster_nu_filename in fna_file_list:
try:
# extract GC_00002 from path/GC_00002.aln
clusterID = gene_cluster_nu_filename.split('/')[-1].split('.')[0]
start = time.time();
geneDiversity_file = open(alignFile_path+'gene_diversity.txt', 'a')
if len( read_fasta(gene_cluster_nu_filename) )==1: # nothing to do for singletons
## na_aln.fa
gene_cluster_nu_aln_filename= gene_cluster_nu_filename.replace('.fna','_na_aln.fa')
## geneSeqID separator '|' is replaced by '-' for msa viewer compatibility
with open(gene_cluster_nu_aln_filename,'wb') as write_file:
for SeqID, Sequence in read_fasta(gene_cluster_nu_filename).iteritems():
write_in_fa(write_file, SeqID.replace('|','-'), Sequence)
os.system( ' '.join(['cp',gene_cluster_nu_aln_filename,gene_cluster_nu_aln_filename.replace('_aln','_aln_reduced')]) )
## aa_aln.fa
gene_cluster_aa_filename= gene_cluster_nu_filename.replace('.fna','.faa')
gene_cluster_aa_aln_filename= gene_cluster_nu_filename.replace('.fna','_aa_aln.fa')
## geneSeqID separator '|' is replaced by '-' for msa viewer compatibility
with open(gene_cluster_aa_aln_filename,'wb') as write_file:
for SeqID, Sequence in read_fasta(gene_cluster_aa_filename).iteritems():
write_in_fa(write_file, SeqID.replace('|','-'), Sequence)
os.system( ' '.join(['cp',gene_cluster_aa_aln_filename,gene_cluster_aa_aln_filename.replace('_aln','_aln_reduced')]) )
geneDiversity_file.write('%s\t%s\n'%(clusterID,'0.0'))
else: # align and build tree
#print gene_cluster_nu_filename
myTree = mpm_tree(gene_cluster_nu_filename)
myTree.codon_align()
myTree.translate()
if simple_tree==False:
myTree.build(raxml=False,fasttree_program=fasttree_name,treetime_used=True)
myTree.ancestral(translate_tree=True)
myTree.refine()
else:
myTree.build(raxml=False,fasttree_program=fasttree_name,treetime_used=False)
myTree.diversity_statistics_nuc()
myTree.export(path=alignFile_path)
#myTree.diversity_statistics_aa()
#random_alnID=myTree.seqs.keys()[0].split('-')[0]
diversity_nuc= round(myTree.diversity_nuc,3)#diversity_aa=round(myTree.diversity_aa,3)
#bestSplit_paraNodes,bestSplit_branchLen = myTree.paralogy_statistics()
#mean_seqLen, std_seqLen= myTree.mean_std_seqLen()
#mean_seqLen, std_seqLen= [ round(i,3) for i in mean_seqLen, std_seqLen ]
geneDiversity_file.write('%s\t%s\n'%(clusterID,diversity_nuc))
if 0:
cluster_correl_stats_file = open(alignFile_path+'cluster_correl_stats.txt', 'a')
cluster_correl_stats_file.write('%s\n'%'\t'.join([
str(i) for i in [clusterID, random_alnID, diversity_nuc, \
mean_seqLen, std_seqLen, bestSplit_paraNodes, bestSplit_branchLen ] ]))
except:
print("Aligning and tree building of %s failed"%gene_cluster_nu_filename)
def align_and_makeTree( fna_file_list, alignFile_path, simple_tree):
fasttree_name= 'fasttree' if check_dependency('fasttree') else 'FastTree'
for gene_cluster_nu_filename in fna_file_list:
try:
# extract GC_00002 from path/GC_00002.aln
clusterID = gene_cluster_nu_filename.split('/')[-1].split('.')[0]
start = time.time();
geneDiversity_file = open(alignFile_path+'gene_diversity.txt', 'a')
if len( read_fasta(gene_cluster_nu_filename) )==1: # nothing to do for singletons
## na_aln.fa
gene_cluster_nu_aln_filename= gene_cluster_nu_filename.replace('.fna','_na_aln.fa')
## geneSeqID separator '|' is replaced by '-' for msa viewer compatibility
with open(gene_cluster_nu_aln_filename,'wb') as write_file:
for SeqID, Sequence in read_fasta(gene_cluster_nu_filename).iteritems():
write_in_fa(write_file, SeqID.replace('|','-'), Sequence)
os.system( ' '.join(['cp',gene_cluster_nu_aln_filename,gene_cluster_nu_aln_filename.replace('_aln','_aln_reduced')]) )
## aa_aln.fa
gene_cluster_aa_filename= gene_cluster_nu_filename.replace('.fna','.faa')
gene_cluster_aa_aln_filename= gene_cluster_nu_filename.replace('.fna','_aa_aln.fa')
## geneSeqID separator '|' is replaced by '-' for msa viewer compatibility
with open(gene_cluster_aa_aln_filename,'wb') as write_file:
for SeqID, Sequence in read_fasta(gene_cluster_aa_filename).iteritems():
write_in_fa(write_file, SeqID.replace('|','-'), Sequence)
os.system( ' '.join(['cp',gene_cluster_aa_aln_filename,gene_cluster_aa_aln_filename.replace('_aln','_aln_reduced')]) )
geneDiversity_file.write('%s\t%s\n'%(clusterID,'0.0'))
else: # align and build tree
#print gene_cluster_nu_filename
myTree = mpm_tree(gene_cluster_nu_filename)
myTree.codon_align()
myTree.translate()
if simple_tree==False:
myTree.build(raxml=False,fasttree_program=fasttree_name,treetime_used=True)
myTree.ancestral(translate_tree=True)
myTree.refine()
else:
myTree.build(raxml=False,fasttree_program=fasttree_name,treetime_used=False)
myTree.diversity_statistics_nuc()
myTree.export(path=alignFile_path)
#myTree.diversity_statistics_aa()
#random_alnID=myTree.seqs.keys()[0].split('-')[0]
diversity_nuc= round(myTree.diversity_nuc,3)#diversity_aa=round(myTree.diversity_aa,3)
#bestSplit_paraNodes,bestSplit_branchLen = myTree.paralogy_statistics()
#mean_seqLen, std_seqLen= myTree.mean_std_seqLen()
#mean_seqLen, std_seqLen= [ round(i,3) for i in mean_seqLen, std_seqLen ]
geneDiversity_file.write('%s\t%s\n'%(clusterID,diversity_nuc))
if 0:
cluster_correl_stats_file = open(alignFile_path+'cluster_correl_stats.txt', 'a')
cluster_correl_stats_file.write('%s\n'%'\t'.join([
str(i) for i in [clusterID, random_alnID, diversity_nuc, \
mean_seqLen, std_seqLen, bestSplit_paraNodes, bestSplit_branchLen ] ]))
except:
print("Aligning and tree building of %s failed"%gene_cluster_nu_filename)
def make_genepresence_alignment(path, disable_gain_loss,
merged_gain_loss_output):
'''
loop over all gene clusters and append 0/1 to strain specific
string used as pseudo alignment of gene presence absence
'''
geneClusterPath = '%s%s' % (path, 'protein_fna/diamond_matches/')
output_path = '%s%s' % (path, 'geneCluster/')
## load strain list and prepare for gene presence/absence
strain_list = load_pickle('%s%s' % (path, 'strain_list.cpk'))
set_totalStrain = set([istrain for istrain in strain_list])
totalStrain = len(set_totalStrain)
dt_strainGene = defaultdict(str)
sorted_genelist = load_sorted_clusters(path)
## sorted_genelist: [(clusterID, [ count_strains,[memb1,...],count_genes]),...]
for clusterID, gene in sorted_genelist:
## append 0/1 to each strain
create_genePresence(dt_strainGene, totalStrain, set_totalStrain,
gene[1])
with open('%s%s' % (output_path, 'genePresence.aln'),
'wb') as presence_outfile:
for istkey in dt_strainGene:
write_in_fa(presence_outfile, istkey, dt_strainGene[istkey])
write_pickle('%s%s' % (output_path, 'dt_genePresence.cpk'), dt_strainGene)
if disable_gain_loss:
geneEvents_dt = {i: 0 for i in range(len(sorted_genelist))}
write_pickle('%s%s' % (output_path, 'dt_geneEvents.cpk'),
geneEvents_dt)
if merged_gain_loss_output:
gene_loss_fname = '%s%s' % (output_path, 'geneGainLossEvent.json')
write_json(dt_strainGene, gene_loss_fname, indent=1)
else:
## strainID as key, presence pattern as value (converted into np.array)
keylist = dt_strainGene.keys()
keylist.sort()
strainID_keymap = {ind: k
for ind, k in enumerate(keylist)
} # dict(zip(keylist, range(3)))
presence_arr = np.array([
np.array(dt_strainGene[k], 'c') for k in keylist
]) # 0: present, 3: absent
presence_arr[presence_arr == '1'] = '3'
for ind, (clusterID, gene) in enumerate(sorted_genelist):
pattern_dt = {
strainID_keymap[strain_ind]: str(patt)
for strain_ind, patt in enumerate(presence_arr[:, ind])
}
pattern_fname = '%s%s_patterns.json' % (output_path, clusterID)
write_json(pattern_dt, pattern_fname, indent=1)
def export_cluster_seq_tmp(cluster_seqs_path, geneCluster_dt,
geneID_to_geneSeqID_dict, gene_na_dict, gene_aa_dict):
""" write nuc/aa sequences for each cluster """
for clusterID, gene in geneCluster_dt.iteritems():
## geneCluster file name
gene_cluster_nu_filename="%s%s"%(clusterID,'.fna')
with open( cluster_seqs_path+gene_cluster_nu_filename, 'wb') as gene_cluster_nu_write:
## write nucleotide sequences into geneCluster files
for gene_memb in gene[1]:
## gene_name format: strain_1|locusTag
strain_name= gene_memb.split('|')[0]
geneSeqID=geneID_to_geneSeqID_dict[gene_memb]
write_in_fa(gene_cluster_nu_write, geneSeqID, gene_na_dict[strain_name][gene_memb] )
def concatenate_core_gene_alignments(input_path, output_path):
core_genes_dt=defaultdict(str)
with open(input_path+'/geneCluster/core_geneList.txt') as core_list:
# all core gene alignments in FASTA files
for gene in core_list:
gene_path= input_path+'/vis/geneCluster/'+gene.rstrip()+'.gz'
with gzip.open(gene_path, 'rb') as zip_file:
for record in SeqIO.parse(zip_file, "fasta"):
#NC_018495-CM9_RS00390-1-hypothetical_protein
accession=record.id.split('-')[0]
core_genes_dt[accession]= '%s%s'%(core_genes_dt[accession], record.seq)
with open(output_path,'wb') as output_file:
for gene_id, gene_seq in core_genes_dt.iteritems():
write_in_fa(output_file, gene_id, gene_seq)
def export_cluster_seq_tmp(cluster_seqs_path, geneCluster_dt,
geneID_to_geneSeqID_dict, gene_na_dict,
gene_aa_dict):
""" write nuc/aa sequences for each cluster """
for clusterID, gene in geneCluster_dt.iteritems():
## geneCluster file name
gene_cluster_nu_filename = "%s%s" % (clusterID, '.fna')
with open(cluster_seqs_path + gene_cluster_nu_filename,
'wb') as gene_cluster_nu_write:
## write nucleotide sequences into geneCluster files
for gene_memb in gene[1]:
## gene_name format: strain_1|locusTag
strain_name = gene_memb.split('|')[0]
geneSeqID = geneID_to_geneSeqID_dict[gene_memb]
write_in_fa(gene_cluster_nu_write, geneSeqID,
gene_na_dict[strain_name][gene_memb])
def concatenate_core_gene_alignments(input_path, output_path):
core_genes_dt = defaultdict(str)
with open(input_path + '/geneCluster/core_geneList.txt') as core_list:
# all core gene alignments in FASTA files
for gene in core_list:
gene_path = input_path + '/vis/geneCluster/' + gene.rstrip(
) + '.gz'
with gzip.open(gene_path, 'rb') as zip_file:
for record in SeqIO.parse(zip_file, "fasta"):
#NC_018495-CM9_RS00390-1-hypothetical_protein
accession = record.id.split('-')[0]
core_genes_dt[accession] = '%s%s' % (
core_genes_dt[accession], record.seq)
with open(output_path, 'wb') as output_file:
for gene_id, gene_seq in core_genes_dt.iteritems():
write_in_fa(output_file, gene_id, gene_seq)
def single_RNACluster_align_and_makeTree(fa_files_list, alignFile_path,
simple_tree):
fasttree_name = 'fasttree' if check_dependency('fasttree') else 'FastTree'
for RNA_cluster_nu_filename in fa_files_list:
if 1: #try:
# extract GC_RNA002 from path/GC_RNA002.aln
clusterID = RNA_cluster_nu_filename.split('/')[-1].split('.')[0]
geneDiversity_file = open(alignFile_path + 'gene_diversity.txt',
'a')
if len(read_fasta(RNA_cluster_nu_filename)
) == 1: # nothing to do for singletons
## na.aln
RNA_cluster_nu_aln_filename = RNA_cluster_nu_filename.replace(
'.fna', '_na.aln')
## RNA SeqID separator '|' is replaced by '-' for msa viewer compatibility
with open(RNA_cluster_nu_aln_filename, 'wb') as write_file:
for SeqID, Sequence in read_fasta(
RNA_cluster_nu_filename).iteritems():
write_in_fa(write_file, SeqID.replace('|', '-'),
Sequence)
geneDiversity_file.write('%s\t%s\n' % (clusterID, '0.0'))
else: # align and build tree
print RNA_cluster_nu_filename
myTree = mpm_tree(RNA_cluster_nu_filename)
myTree.align()
if simple_tree == False:
myTree.build(raxml=False,
fasttree_program=fasttree_name,
treetime_used=True)
myTree.ancestral(translate_tree=True)
myTree.refine()
else:
myTree.build(raxml=False,
fasttree_program=fasttree_name,
treetime_used=False)
myTree.diversity_statistics_nuc()
myTree.export(path=alignFile_path, RNA_specific=True)
RNA_diversity_values = '{0:.3f}'.format(myTree.diversity_nuc)
geneDiversity_file.write('%s\t%s\n' %
(clusterID, RNA_diversity_values))
print clusterID, RNA_diversity_values
if 0: #except:
print("Aligning and tree building of RNA %s failed" %
RNA_cluster_nu_filename)
def make_genepresence_alignment(path, disable_gain_loss, merged_gain_loss_output):
'''
loop over all gene clusters and append 0/1 to strain specific
string used as pseudo alignment of gene presence absence
'''
geneClusterPath='%s%s'%(path,'protein_fna/diamond_matches/')
output_path='%s%s'%(path,'geneCluster/');
## load strain list and prepare for gene presence/absence
strain_list= load_pickle('%s%s'%(path,'strain_list.cpk'))
set_totalStrain=set([ istrain for istrain in strain_list ])
totalStrain=len(set_totalStrain)
dt_strainGene= defaultdict(str)
sorted_genelist = load_sorted_clusters(path)
## sorted_genelist: [(clusterID, [ count_strains,[memb1,...],count_genes]),...]
for clusterID, gene in sorted_genelist:
## append 0/1 to each strain
create_genePresence(dt_strainGene, totalStrain, set_totalStrain, gene[1])
with open('%s%s'%(output_path,'genePresence.aln'),'wb') as presence_outfile:
for istkey in dt_strainGene:
write_in_fa( presence_outfile, istkey, dt_strainGene[istkey])
write_pickle('%s%s'%(output_path,'dt_genePresence.cpk'), dt_strainGene)
if disable_gain_loss:
geneEvents_dt={ i:0 for i in range(len(sorted_genelist)) }
write_pickle('%s%s'%(output_path,'dt_geneEvents.cpk'), geneEvents_dt)
if merged_gain_loss_output:
gene_loss_fname='%s%s'%(output_path,'geneGainLossEvent.json')
write_json(dt_strainGene, gene_loss_fname, indent=1)
else:
## strainID as key, presence pattern as value (converted into np.array)
keylist= dt_strainGene.keys(); keylist.sort()
strainID_keymap= {ind:k for ind, k in enumerate(keylist)} # dict(zip(keylist, range(3)))
presence_arr= np.array([ np.array(dt_strainGene[k],'c') for k in keylist]) # 0: present, 3: absent
presence_arr[presence_arr=='1']='3'
for ind, (clusterID, gene) in enumerate(sorted_genelist):
pattern_dt= { strainID_keymap[strain_ind]:str(patt) for strain_ind, patt in enumerate(presence_arr[:, ind])}
pattern_fname='%s%s_patterns.json'%(output_path,clusterID)
write_json(pattern_dt, pattern_fname, indent=1)
def create_core_SNP_matrix(path, core_cutoff=1.0, core_gene_strain_fpath=''):#1.0
""" create SNP matrix using core gene SNPs
input: strain_list.cpk, core_geneList.cpk
output: SNP_whole_matrix.aln
core_cutoff: percentage of strains used to decide whether a gene is core
default: 1.0 (strictly core gene, which is present in all strains)
customized: 0.9 ( soft core, considered as core if present in 90% of strains)
"""
import os,sys,operator
import numpy as np
import numpy.ma as ma
from collections import defaultdict
from sf_miscellaneous import read_fasta, write_pickle, load_pickle, write_in_fa
alnFilePath='%s%s'%(path,'geneCluster/')
output_path= alnFilePath
## create core gene list
corelist=[]
strain_list=load_pickle(path+'strain_list.cpk')
totalStrain= len(strain_list)
sorted_geneList = load_sorted_clusters(path)
if core_gene_strain_fpath!='':
with open(core_gene_strain_fpath,'rb') as core_gene_strain_file:
core_strain_set= set([i.rstrip().replace('-','_') for i in core_gene_strain_file])
with open(output_path+'core_geneList.txt','wb') as outfile:
for clusterID, vg in sorted_geneList:
if core_cutoff==1.0:
strain_core_cutoff=totalStrain
else:
strain_core_cutoff=int(totalStrain*core_cutoff)
if vg[0]==vg[2] and vg[0]>=strain_core_cutoff:
coreGeneName='%s%s'%(clusterID,'_na_aln.fa')
## sequences might be discarded because of premature stops
coreGeneName_path= alnFilePath+coreGeneName
if os.path.exists(coreGeneName_path) and len(read_fasta(coreGeneName_path)) >= strain_core_cutoff:
if core_gene_strain_fpath!='' and len(core_strain_set-set([i.split('|')[0] for i in vg[1]]))!=0:
continue
outfile.write(coreGeneName+'\n')
corelist.append(coreGeneName)
else:
#print '%s%s%s'%('warning: ',coreGeneName_path,' is not a core gene')
pass
write_pickle(output_path+'core_geneList.cpk',corelist)
refSeqList=load_pickle(path+'strain_list.cpk');refSeqList.sort()
snp_fre_lst=[]; snp_wh_matrix_flag=0
snp_pos_dt=defaultdict(list); snp_whole_matrix=np.array([])
snps_by_gene=[]
for align_file in corelist:## core genes
nuc_array=np.array([]) # array to store nucleotides for each gene
gene_seq_dt=read_fasta(alnFilePath+align_file)
if core_cutoff!=1.0:
# set sequences for missing gene (space*gene_length)
missing_gene_seq=' '*len(gene_seq_dt.values()[0])
totalStrain_sorted_lst=sorted(strain_list)
# build strain_seq_dt from gene_seq_dt
strain_seq_dt=defaultdict()
for gene, seq in gene_seq_dt.iteritems():
strain_seq_dt[gene.split('-')[0]]=seq # strain-locus_tag-...
strain_seq_sorted_lst=sorted(strain_seq_dt.items(), key=lambda x: x[0])
start_flag=0
if core_cutoff==1.0:
for ka, va in strain_seq_sorted_lst:
if start_flag==0:
nuc_array=np.array(np.fromstring(va, dtype='S1'))
start_flag=1
else:
nuc_array=np.vstack((nuc_array,np.fromstring(va, dtype='S1')))
## find SNP positions
position_polymorphic = np.any(nuc_array != nuc_array[0, :], axis = 0)
position_has_gap = np.any(nuc_array=='-', axis=0)
position_SNP = position_polymorphic&(~position_has_gap)
snp_columns = nuc_array[:,position_SNP]
snp_pos_dt[align_file]=np.where(position_SNP)[0]
else:
## add '-' for missing genes when dealing with soft core genes
core_gene_strain=[ gene for gene in strain_seq_dt.keys()]
for strain in totalStrain_sorted_lst:
if start_flag==0:
if strain in core_gene_strain:
nuc_array=np.array(np.fromstring(strain_seq_dt[strain], dtype='S1'))
else:
print 'Soft core gene: gene absent in strain %s on cluster %s'%(strain,align_file)
nuc_array=np.array(np.fromstring(missing_gene_seq, dtype='S1'))
start_flag=1
else:
if strain in core_gene_strain:
nuc_array=np.vstack((nuc_array,np.fromstring(strain_seq_dt[strain], dtype='S1')))
else:
print 'Soft core gene: gene absent in strain %s on cluster %s'%(strain,align_file)
nuc_array=np.vstack((nuc_array,np.fromstring(missing_gene_seq, dtype='S1')))
## find SNP positions
## mask missing genes -- determine rows that have ' ' in every column
is_missing = np.all(nuc_array==' ',axis=1)
masked_non_missing_array= np.ma.masked_array(nuc_array, nuc_array==' ')
position_polymorphic = np.any(masked_non_missing_array!= masked_non_missing_array[0, :],axis = 0)
position_has_gap = np.any(masked_non_missing_array=='-',axis=0)
position_SNP = position_polymorphic&(~position_has_gap)
# the below seems duplicated from 5 lines above??
if is_missing.sum()>0: # with missing genes
nuc_array[is_missing]='-'
snp_columns = nuc_array[:,position_SNP]
snp_pos_dt[align_file]=np.where(position_SNP)[0]
#print snp_columns
if snp_wh_matrix_flag==0:
snp_whole_matrix=snp_columns;
snp_wh_matrix_flag=1
else:
snp_whole_matrix=np.hstack((snp_whole_matrix, snp_columns))
write_pickle(output_path+'snp_pos.cpk',snp_pos_dt)
with open(output_path+'SNP_whole_matrix.aln','wb') as outfile:
for ind, isw in enumerate(snp_whole_matrix):
write_in_fa( outfile, refSeqList[ind], isw.tostring() )
def create_core_SNP_matrix(path,
core_cutoff=1.0,
core_gene_strain_fpath=''): #1.0
""" create SNP matrix using core gene SNPs
input: strain_list.cpk, core_geneList.cpk
output: SNP_whole_matrix.aln
core_cutoff: percentage of strains used to decide whether a gene is core
default: 1.0 (strictly core gene, which is present in all strains)
customized: 0.9 ( soft core, considered as core if present in 90% of strains)
"""
import os, sys, operator
import numpy as np
import numpy.ma as ma
from collections import defaultdict
from sf_miscellaneous import read_fasta, write_pickle, load_pickle, write_in_fa
alnFilePath = '%s%s' % (path, 'geneCluster/')
output_path = alnFilePath
## create core gene list
corelist = []
strain_list = load_pickle(path + 'strain_list.cpk')
totalStrain = len(strain_list)
sorted_geneList = load_sorted_clusters(path)
if core_gene_strain_fpath != '':
with open(core_gene_strain_fpath, 'rb') as core_gene_strain_file:
core_strain_set = set(
[i.rstrip().replace('-', '_') for i in core_gene_strain_file])
with open(output_path + 'core_geneList.txt', 'wb') as outfile:
for clusterID, vg in sorted_geneList:
if core_cutoff == 1.0:
strain_core_cutoff = totalStrain
else:
strain_core_cutoff = int(totalStrain * core_cutoff)
if vg[0] == vg[2] and vg[0] >= strain_core_cutoff:
coreGeneName = '%s%s' % (clusterID, '_na_aln.fa')
## sequences might be discarded because of premature stops
coreGeneName_path = alnFilePath + coreGeneName
if os.path.exists(coreGeneName_path) and len(
read_fasta(coreGeneName_path)) >= strain_core_cutoff:
if core_gene_strain_fpath != '' and len(
core_strain_set -
set([i.split('|')[0] for i in vg[1]])) != 0:
continue
outfile.write(coreGeneName + '\n')
corelist.append(coreGeneName)
else:
#print '%s%s%s'%('warning: ',coreGeneName_path,' is not a core gene')
pass
write_pickle(output_path + 'core_geneList.cpk', corelist)
refSeqList = load_pickle(path + 'strain_list.cpk')
refSeqList.sort()
snp_fre_lst = []
snp_wh_matrix_flag = 0
snp_pos_dt = defaultdict(list)
snp_whole_matrix = np.array([])
snps_by_gene = []
for align_file in corelist: ## core genes
nuc_array = np.array([]) # array to store nucleotides for each gene
gene_seq_dt = read_fasta(alnFilePath + align_file)
if core_cutoff != 1.0:
# set sequences for missing gene (space*gene_length)
missing_gene_seq = ' ' * len(gene_seq_dt.values()[0])
totalStrain_sorted_lst = sorted(strain_list)
# build strain_seq_dt from gene_seq_dt
strain_seq_dt = defaultdict()
for gene, seq in gene_seq_dt.iteritems():
strain_seq_dt[gene.split('-')[0]] = seq # strain-locus_tag-...
strain_seq_sorted_lst = sorted(strain_seq_dt.items(),
key=lambda x: x[0])
start_flag = 0
if core_cutoff == 1.0:
for ka, va in strain_seq_sorted_lst:
if start_flag == 0:
nuc_array = np.array(np.fromstring(va, dtype='S1'))
start_flag = 1
else:
nuc_array = np.vstack(
(nuc_array, np.fromstring(va, dtype='S1')))
## find SNP positions
position_polymorphic = np.any(nuc_array != nuc_array[0, :], axis=0)
position_has_gap = np.any(nuc_array == '-', axis=0)
position_SNP = position_polymorphic & (~position_has_gap)
snp_columns = nuc_array[:, position_SNP]
snp_pos_dt[align_file] = np.where(position_SNP)[0]
else:
## add '-' for missing genes when dealing with soft core genes
core_gene_strain = [gene for gene in strain_seq_dt.keys()]
for strain in totalStrain_sorted_lst:
if start_flag == 0:
if strain in core_gene_strain:
nuc_array = np.array(
np.fromstring(strain_seq_dt[strain], dtype='S1'))
else:
print 'Soft core gene: gene absent in strain %s on cluster %s' % (
strain, align_file)
nuc_array = np.array(
np.fromstring(missing_gene_seq, dtype='S1'))
start_flag = 1
else:
if strain in core_gene_strain:
nuc_array = np.vstack(
(nuc_array,
np.fromstring(strain_seq_dt[strain], dtype='S1')))
else:
print 'Soft core gene: gene absent in strain %s on cluster %s' % (
strain, align_file)
nuc_array = np.vstack((nuc_array,
np.fromstring(missing_gene_seq,
dtype='S1')))
## find SNP positions
## mask missing genes -- determine rows that have ' ' in every column
is_missing = np.all(nuc_array == ' ', axis=1)
masked_non_missing_array = np.ma.masked_array(
nuc_array, nuc_array == ' ')
position_polymorphic = np.any(
masked_non_missing_array != masked_non_missing_array[0, :],
axis=0)
position_has_gap = np.any(masked_non_missing_array == '-', axis=0)
position_SNP = position_polymorphic & (~position_has_gap)
# the below seems duplicated from 5 lines above??
if is_missing.sum() > 0: # with missing genes
nuc_array[is_missing] = '-'
snp_columns = nuc_array[:, position_SNP]
snp_pos_dt[align_file] = np.where(position_SNP)[0]
#print snp_columns
if snp_wh_matrix_flag == 0:
snp_whole_matrix = snp_columns
snp_wh_matrix_flag = 1
else:
snp_whole_matrix = np.hstack((snp_whole_matrix, snp_columns))
write_pickle(output_path + 'snp_pos.cpk', snp_pos_dt)
with open(output_path + 'SNP_whole_matrix.aln', 'wb') as outfile:
for ind, isw in enumerate(snp_whole_matrix):
write_in_fa(outfile, refSeqList[ind], isw.tostring())
def create_split_cluster_files(file_path, fname, gene_list1, gene_list2,
geneCluster_dt):
"""
delete the old cluster and create two new clusters
params:
new_fa_files: list to which new file names are appeneded
gene_list1/2: lists containing the genes in the new split clusters
geneCluster_dt: cluster dictionary to be updated
"""
orgin_nwk_name = fname.split('/')[-1]
clusterID = orgin_nwk_name.replace('.nwk', '')
origin_cluster_nu_fa = orgin_nwk_name.replace('nwk', 'fna')
origin_cluster_aa_fa = orgin_nwk_name.replace('nwk', 'faa')
split_fa_files_set = set()
## load genes from old clusters
origin_nu_fa_dt = read_fasta(file_path + origin_cluster_nu_fa)
origin_aa_fa_dt = read_fasta(file_path + origin_cluster_aa_fa)
sgs_index = 0
## delete old (split) clusters
try:
#print('deleting:',orgin_nwk_name)
##debug:
##print('deleting:',orgin_nwk_name,gene_list1,gene_list2, clusterID)
del geneCluster_dt[clusterID]
with open(file_path + 'old_clusters_paralogSplit.txt',
'a') as delete_cluster_file:
delete_cluster_file.write('%s\n' % clusterID)
if os.path.exists(fname):
suffix_list = [
'_aa_aln.fa', '_na_aln.fa', '.fna', '.faa', '.nwk',
'_tree.json'
]
else:
suffix_list = ['_aa_aln.fa', '_na_aln.fa', '.fna', '.faa']
tmp_files = ' '.join(
[file_path + clusterID + suffix for suffix in suffix_list])
command_move_deleted_clusters = ' '.join(
['mv', tmp_files, file_path + 'paralog_splits/'])
os.system(command_move_deleted_clusters)
except:
print("paralog splitting: can't delete", orgin_nwk_name)
##debug:
##print("can't delete",orgin_nwk_name,gene_list1,gene_list2, clusterID)
## write new cluster fa files
## split_gene_list has geneSeqID instead of geneID
for split_gene_list in (list(gene_list1), list(gene_list2)):
sgs_index += 1
newClusterId = "%s_p%s" % (clusterID, sgs_index)
gene_cluster_nu_filename = "%s%s" % (newClusterId, '.fna')
gene_cluster_aa_filename = "%s%s" % (newClusterId, '.faa')
gene_cluster_nu_write = open(file_path + gene_cluster_nu_filename,
'wb')
gene_cluster_aa_write = open(file_path + gene_cluster_aa_filename,
'wb')
split_fa_files_set |= set([file_path + gene_cluster_nu_filename])
## write new split cluster files
for gene_memb in split_gene_list:
if "\\'" in gene_memb:
gene_memb = gene_memb.replace("\\'", "'")
try:
write_in_fa(gene_cluster_nu_write, gene_memb,
origin_nu_fa_dt[gene_memb])
write_in_fa(gene_cluster_aa_write, gene_memb,
origin_aa_fa_dt[gene_memb])
except:
print 'paralogy splitting (problem to write new split cluster files)', fname #, gene_memb, gene_list1, gene_list2
gene_cluster_nu_write.close()
gene_cluster_aa_write.close()
geneCluster_dt[newClusterId] = [0, [], 0]
## num_stains
geneCluster_dt[newClusterId][0] = len(
dict(Counter([ig.split('|')[0] for ig in split_gene_list])).keys())
## num_genes
geneCluster_dt[newClusterId][2] = len(
dict(Counter([ig for ig in split_gene_list])).keys())
## gene members
geneCluster_dt[newClusterId][1] = [
ig.split('-')[0] for ig in split_gene_list
]
return split_fa_files_set
def output_cutted_clusters(file_path, uncluster_filename, gene_list, cut_branch_threshold, treefile_used=None, cut_leftover=None):
"""
delete the unclustered file and create new clusters
params:
gene_list: lists containing the genes in the new split clusters
geneCluster_dt: cluster dictionary to be updated
cut_leftover: flag to indicate whether there are the leftover nodes
after cutting long branches. Default: empty.
"""
clusterID = uncluster_filename.replace('.fna','')
origin_uncluster_nu_fa = uncluster_filename
origin_uncluster_aa_fa = uncluster_filename.replace('fna','faa')
new_fa_files=set()
## load origin cluster fa files
origin_nu_fa_dt = read_fasta(file_path+origin_uncluster_nu_fa)
origin_aa_fa_dt = read_fasta(file_path+origin_uncluster_aa_fa)
## split_gene_list has geneSeqID instead of geneID
for sgs_index,split_gene_list in enumerate(gene_list,1):
if cut_leftover==True:
## newClusterId for the rest genes (_r as identifier)
newClusterId="%s_r%s"%(clusterID,sgs_index)
else:
newClusterId="%s_%s"%(clusterID,sgs_index)
#=============================================
## write new divided/split cluster files
gene_cluster_nu_filename="%s%s"%(newClusterId,'.fna')
gene_cluster_nu_filepath= file_path+gene_cluster_nu_filename
gene_cluster_nu_write=open(gene_cluster_nu_filepath , 'wb')
gene_cluster_aa_filename="%s%s"%(newClusterId,'.faa')
gene_cluster_aa_filepath= file_path+gene_cluster_aa_filename
gene_cluster_aa_write=open( file_path+gene_cluster_aa_filename, 'wb')
for gene_memb in split_gene_list:
if "\\'" in gene_memb: # Replace '\' in node name:
## NC_018495|CM9_RS01675-1-guanosine-3',5'-... in fasta ID
## 'NC_018495|CM9_RS01675-1-guanosine-3\',5\'-...' in nwk node name
## Use origin_nu_fa_dt[gene_memb] will throw the KeyError:
## "NC_018495|CM9_RS01675-1-guanosine-3\\',5\\'"
gene_memb=gene_memb.replace("\\'","'")
write_in_fa(gene_cluster_nu_write, gene_memb, origin_nu_fa_dt[gene_memb])
write_in_fa(gene_cluster_aa_write, gene_memb, origin_aa_fa_dt[gene_memb])
gene_cluster_nu_write.close(); gene_cluster_aa_write.close();
#=============================================
if cut_leftover==True:
## align the rest genes, build tree, cut long branches till nothing can be cutted.
cutTree_outputCluster([gene_cluster_nu_filepath],file_path, cut_branch_threshold, treefile_used)
else:
## record the misclusters to be deleted (already addressed in cutTree_outputCluster )
## it will output the same cluster several times
#with open(file_path+'old_clusters_longSplit.txt', 'a') as delete_cluster_file:
# delete_cluster_file.write('%s\n'%uncluster_filename)
## add record in new_clusters_longSplit.txt, which is used for align new clusters
new_fa_files.add(gene_cluster_nu_filepath)
## write cluster statistics in folder update_long_branch_splits
addin_geneCluster_dt=defaultdict(list)
addin_geneCluster_dt[ newClusterId ] = [0,[],0]
## num_stains
addin_geneCluster_dt[ newClusterId ][0]=len(dict(Counter([ ig.split('|')[0] for ig in split_gene_list])).keys())
## num_genes
addin_geneCluster_dt[ newClusterId ][2]=len(dict(Counter([ ig for ig in split_gene_list])).keys())
## gene members
addin_geneCluster_dt[ newClusterId ][1]=[ ig.split('-')[0] for ig in split_gene_list ]
## cPickle new cluster statistics
write_pickle(''.join([file_path,'update_long_branch_splits/', newClusterId,'.cpk']),addin_geneCluster_dt)
## write records in gene_diversity file
with open(file_path+'new_clusters_longSplit.txt', 'a') as refined_cluster_file:
for i in new_fa_files:
refined_cluster_file.write('%s\n'%i)
def output_cutted_clusters(file_path,
uncluster_filename,
gene_list,
cut_branch_threshold,
treefile_used=None,
cut_leftover=None):
"""
delete the unclustered file and create new clusters
params:
gene_list: lists containing the genes in the new split clusters
geneCluster_dt: cluster dictionary to be updated
cut_leftover: flag to indicate whether there are the leftover nodes
after cutting long branches. Default: empty.
"""
clusterID = uncluster_filename.replace('.fna', '')
origin_uncluster_nu_fa = uncluster_filename
origin_uncluster_aa_fa = uncluster_filename.replace('fna', 'faa')
new_fa_files = set()
## load origin cluster fa files
origin_nu_fa_dt = read_fasta(file_path + origin_uncluster_nu_fa)
origin_aa_fa_dt = read_fasta(file_path + origin_uncluster_aa_fa)
## split_gene_list has geneSeqID instead of geneID
for sgs_index, split_gene_list in enumerate(gene_list, 1):
if cut_leftover == True:
## newClusterId for the rest genes (_r as identifier)
newClusterId = "%s_r%s" % (clusterID, sgs_index)
else:
newClusterId = "%s_%s" % (clusterID, sgs_index)
#=============================================
## write new divided/split cluster files
gene_cluster_nu_filename = "%s%s" % (newClusterId, '.fna')
gene_cluster_nu_filepath = file_path + gene_cluster_nu_filename
gene_cluster_nu_write = open(gene_cluster_nu_filepath, 'wb')
gene_cluster_aa_filename = "%s%s" % (newClusterId, '.faa')
gene_cluster_aa_filepath = file_path + gene_cluster_aa_filename
gene_cluster_aa_write = open(file_path + gene_cluster_aa_filename,
'wb')
for gene_memb in split_gene_list:
if "\\'" in gene_memb: # Replace '\' in node name:
## NC_018495|CM9_RS01675-1-guanosine-3',5'-... in fasta ID
## 'NC_018495|CM9_RS01675-1-guanosine-3\',5\'-...' in nwk node name
## Use origin_nu_fa_dt[gene_memb] will throw the KeyError:
## "NC_018495|CM9_RS01675-1-guanosine-3\\',5\\'"
gene_memb = gene_memb.replace("\\'", "'")
write_in_fa(gene_cluster_nu_write, gene_memb,
origin_nu_fa_dt[gene_memb])
write_in_fa(gene_cluster_aa_write, gene_memb,
origin_aa_fa_dt[gene_memb])
gene_cluster_nu_write.close()
gene_cluster_aa_write.close()
#=============================================
if cut_leftover == True:
## align the rest genes, build tree, cut long branches till nothing can be cutted.
cutTree_outputCluster([gene_cluster_nu_filepath], file_path,
cut_branch_threshold, treefile_used)
else:
## record the misclusters to be deleted (already addressed in cutTree_outputCluster )
## it will output the same cluster several times
#with open(file_path+'old_clusters_longSplit.txt', 'a') as delete_cluster_file:
# delete_cluster_file.write('%s\n'%uncluster_filename)
## add record in new_clusters_longSplit.txt, which is used for align new clusters
new_fa_files.add(gene_cluster_nu_filepath)
## write cluster statistics in folder update_long_branch_splits
addin_geneCluster_dt = defaultdict(list)
addin_geneCluster_dt[newClusterId] = [0, [], 0]
## num_stains
addin_geneCluster_dt[newClusterId][0] = len(
dict(Counter([ig.split('|')[0]
for ig in split_gene_list])).keys())
## num_genes
addin_geneCluster_dt[newClusterId][2] = len(
dict(Counter([ig for ig in split_gene_list])).keys())
## gene members
addin_geneCluster_dt[newClusterId][1] = [
ig.split('-')[0] for ig in split_gene_list
]
## cPickle new cluster statistics
write_pickle(
''.join([
file_path, 'update_long_branch_splits/', newClusterId,
'.cpk'
]), addin_geneCluster_dt)
## write records in gene_diversity file
with open(file_path + 'new_clusters_longSplit.txt',
'a') as refined_cluster_file:
for i in new_fa_files:
refined_cluster_file.write('%s\n' % i)
def create_split_cluster_files(file_path, fname,
gene_list1, gene_list2, geneCluster_dt):
"""
delete the old cluster and create two new clusters
params:
new_fa_files: list to which new file names are appeneded
gene_list1/2: lists containing the genes in the new split clusters
geneCluster_dt: cluster dictionary to be updated
"""
orgin_nwk_name = fname.split('/')[-1]
clusterID = orgin_nwk_name.replace('.nwk','')
origin_cluster_nu_fa = orgin_nwk_name.replace('nwk','fna')
origin_cluster_aa_fa = orgin_nwk_name.replace('nwk','faa')
split_fa_files_set=set()
## load genes from old clusters
origin_nu_fa_dt = read_fasta(file_path+origin_cluster_nu_fa)
origin_aa_fa_dt = read_fasta(file_path+origin_cluster_aa_fa)
sgs_index=0
## delete old (split) clusters
try:
#print('deleting:',orgin_nwk_name)
##debug:
##print('deleting:',orgin_nwk_name,gene_list1,gene_list2, clusterID)
del geneCluster_dt[clusterID]
with open(file_path+'old_clusters_paralogSplit.txt', 'a') as delete_cluster_file:
delete_cluster_file.write('%s\n'%clusterID)
if os.path.exists(fname):
suffix_list=['_aa_aln.fa','_na_aln.fa','.fna','.faa','.nwk','_tree.json']
else:
suffix_list=['_aa_aln.fa','_na_aln.fa','.fna','.faa']
tmp_files=' '.join([ file_path+clusterID+suffix for suffix in suffix_list ])
command_move_deleted_clusters=' '.join(['mv', tmp_files, file_path+'paralog_splits/'])
os.system(command_move_deleted_clusters)
except:
print("paralog splitting: can't delete",orgin_nwk_name)
##debug:
##print("can't delete",orgin_nwk_name,gene_list1,gene_list2, clusterID)
## write new cluster fa files
## split_gene_list has geneSeqID instead of geneID
for split_gene_list in (list(gene_list1), list(gene_list2)):
sgs_index+=1
newClusterId="%s_p%s"%(clusterID,sgs_index)
gene_cluster_nu_filename="%s%s"%(newClusterId,'.fna')
gene_cluster_aa_filename="%s%s"%(newClusterId,'.faa')
gene_cluster_nu_write=open( file_path+gene_cluster_nu_filename, 'wb')
gene_cluster_aa_write=open( file_path+gene_cluster_aa_filename, 'wb')
split_fa_files_set |= set([file_path+gene_cluster_nu_filename])
## write new split cluster files
for gene_memb in split_gene_list:
if "\\'" in gene_memb:
gene_memb=gene_memb.replace("\\'","'")
try:
write_in_fa(gene_cluster_nu_write, gene_memb, origin_nu_fa_dt[gene_memb])
write_in_fa(gene_cluster_aa_write, gene_memb, origin_aa_fa_dt[gene_memb])
except:
print 'paralogy splitting (problem to write new split cluster files)', fname #, gene_memb, gene_list1, gene_list2
gene_cluster_nu_write.close(); gene_cluster_aa_write.close();
geneCluster_dt[ newClusterId ] = [0,[],0]
## num_stains
geneCluster_dt[ newClusterId ][0]=len(dict(Counter([ ig.split('|')[0] for ig in split_gene_list])).keys())
## num_genes
geneCluster_dt[ newClusterId ][2]=len(dict(Counter([ ig for ig in split_gene_list])).keys())
## gene members
geneCluster_dt[ newClusterId ][1]=[ ig.split('-')[0] for ig in split_gene_list ]
return split_fa_files_set
def gbk_translation(strainID, gbk_fname, protein_fname, nucleotide_fname, RNA_fname,
geneID_to_geneSeqID_dict,geneID_to_description_dict,
RNAID_to_SeqID_dict, RNAID_to_description_dict,
gene_aa_dict, gene_na_dict, RNA_dict, enable_RNA_clustering):
'''
extract sequences and meta informations of all genes in one reference genbank file
params:
- gbk_fname: Genbank filename
- protein_fname: file into which all amino acid sequences are written
in fasta format. needed as input for diamond
- nucleotide_fname: file into which all nucleotide sequences are written
in fasta format. needed for cluster sequences
- RNA_fname: RNA nucleotide_sequences are written in fasta format.
Needed as RNA_blast_input
- geneID_to_geneSeqID_dict: dictionary linking geneID to gene sequence ID
modified in place (key: geneID; value: geneSeqID )
- geneID_to_description_dict: dictionary linking geneID to description info
modified in place (key: geneID; value: a dict including
information on contig_index, annotation or more)
- RNAID_to_SeqID_dict: dictionary linking RNAID to RNA sequence ID
modified in place (key: RNAID; value: SeqID )
- RNAID_to_description_dict: dictionary linking RNAID to description info
modified in place (key: RNAID; value: a dict including
information on contig_index, annotation or more)
- enable_RNA_clustering: not cluster rRNA
'''
aa_sequence_file=open(protein_fname, 'wb')
nu_sequence_file=open(nucleotide_fname, 'wb')
if enable_RNA_clustering:
RNA_sequence_file=open(RNA_fname, 'wb')
contig_index=0
check_CDS_passed=0
for contig in SeqIO.parse(gbk_fname,'genbank'):
contig_index+=1
for feature in contig.features:
if feature.type=='CDS':
if not check_CDS_passed:
check_CDS_passed=1
if 'product' in feature.qualifiers and 'translation' in feature.qualifiers :
if 'gene' in feature.qualifiers :
geneName='%s'%(feature.qualifiers['gene'][0]).replace(' ','_')
else: geneName=''
product=feature.qualifiers['product'][0]
annotation= '_'.join(product.split(' '))
trans_seq=feature.qualifiers['translation'][0]
if 'locus_tag' in feature.qualifiers:
locus_tag=feature.qualifiers['locus_tag'][0]
else:
locus_tag=feature.qualifiers['db_xref'][0].split(':')[1]
## force to replace '-' with '_' in locus_tag
if '-' in locus_tag:
locus_tag=locus_tag.replace('-','_')
if "PROKKA" in locus_tag:
locus_tag=locus_tag.replace('PROKKA_','')
if '%s_'%strainID in locus_tag:
locus_tag=locus_tag.split('%s_'%strainID)[1]
## geneID is composed of strain_name and locus_tag
## Keeping '|' separator is important, which is used later in orthAgogue.
geneID= '%s|%s'%(strainID,locus_tag)
na_seq=str(feature.extract(contig.seq))
write_in_fa(aa_sequence_file, geneID, trans_seq)
write_in_fa(nu_sequence_file, geneID, na_seq)
gene_aa_dict[strainID][geneID]=trans_seq
gene_na_dict[strainID][geneID]=na_seq
# give tag 'gname:' to genes which have gene name and separate it from annotation
geneID_to_description_dict[geneID]={'geneName': geneName,
'contig': contig_index,
'annotation': annotation}
if geneName!='':
geneName='%s_'%geneName
geneID_to_geneSeqID_dict[geneID]='%s|%s-%d-%s%s'%(strainID,
locus_tag, contig_index,
geneName, annotation)
elif enable_RNA_clustering and (feature.type=='rRNA'):
#elif not enable_RNA_clustering and (feature.type=='rRNA' or feature.type=='tRNA'):
if 'product' in feature.qualifiers:
geneName=''
product=feature.qualifiers['product'][0]
annotation= '_'.join(product.split(' '))
try:
locus_tag=feature.qualifiers['locus_tag'][0]
except: # make a random string when locus_tag absent
locus_tag=time.strftime('%S',time.gmtime())+str(random.randint(0,10000000))
if "PROKKA" in locus_tag:
locus_tag=locus_tag.replace('PROKKA_','')
if '%s_'%strainID in locus_tag:
locus_tag=locus_tag.split('%s_'%strainID)[1]
## RNA is composed of strain_name and locus_tag
## Keeping '|' separator is important, which is used later in orthAgogue.
RNAID= '%s|%s'%(strainID,locus_tag)
RNA_seq= str(feature.extract(contig.seq))
write_in_fa(RNA_sequence_file, RNAID, RNA_seq)
RNA_dict[strainID][RNAID]=RNA_seq
# give tag 'gname:' to genes which have gene name and separate it from annotation
RNAID_to_description_dict[RNAID]={
'geneName': '',
'contig': contig_index,
'annotation': annotation}
RNAID_to_SeqID_dict[RNAID]='%s|%s-%d-%s%s'%(strainID,
locus_tag, contig_index,
geneName, annotation)
aa_sequence_file.close(); nu_sequence_file.close()
return check_CDS_passed