def buildAltMouseExonAnnotations(species,array_type,force,genomic_build):
"""Code required to:
1) Extract out Affymetrix provided exon sequence (probeset sequence extracted from "probeset_sequence_reversed.txt", derived
directly from the Affymetrix AltMouse probe annotation file), from the "SEQUENCE-transcript-dbase.txt" (built using
dump-chip1 .gff sequence and AltMerge-Peptide Informatics script "sequence_analysis_AltMouse_refseq.py").
2) Once exported, grab full length exon sequences using exon/intron coordinates matches to full-length gene sequences with 2kb
flanking sequence to efficiently predict microRNA binding site exclusion (reAnnotateCriticalExonSequences) and later for
coordinate mapping to get exons aligning with UCSC annotated splicing annotations and exons. This sequence data replaced
the previous file (don't need to re-run this - see rederive_exonseq == 'yes' below for reference).
3) Match the updated exon sequences to the most recent genomic coordinates and build the exact equivalent of the exon array
Mm_Ensembl_probeset.txt database (same structure and ExonArrayEnsemblRules.py code). This involves running EnsemblImport.
This code should be run before the exon array location build code since the "Mm_Ensembl_probeset.txt" is created and then re-
written as "Mm_AltMouse_Ensembl_probeset.txt".
"""
import JunctionArray
import JunctionArrayEnsemblRules
rederive_exonseq = 'no'
### Only needs to be run once, to export exon sequence for AltMouse array the original (1 and 2 above)
if rederive_exonseq == 'yes':
import AltAnalyze
import ExonAnnotate_module
import ExonAnalyze_module
agglomerate_inclusion_probesets = 'no'; onlyAnalyzeJunctions='no'
probeset_annotations_file = "AltDatabase/"+species+"/"+array_type+"/"+"MASTER-probeset-transcript.txt"
verifyFile(probeset_annotations_file,array_type) ### Will force download if missing
exon_db={}; filtered_arrayids={};filter_status='no'
constituitive_probeset_db,exon_db,genes_being_analyzed = AltAnalyze.importSplicingAnnotationDatabase(probeset_annotations_file,array_type,filtered_arrayids,filter_status)
alt_junction_db,critical_exon_db,exon_dbase,exon_inclusion_db,exon_db = ExonAnnotate_module.identifyPutativeSpliceEvents(exon_db,constituitive_probeset_db,{},agglomerate_inclusion_probesets,onlyAnalyzeJunctions)
ExonAnnotate_module.exportJunctionComparisons(alt_junction_db,critical_exon_db,exon_dbase)
print "Finished exporting junctions used in AltMouse array comparisons."
ExonAnalyze_module.exportAltMouseExonSequence()
JunctionArray.reAnnotateCriticalExonSequences(species,array_type)
### Get UCSC associations (download databases if necessary)
mRNA_Type = 'mrna'; run_from_scratch = 'yes'
export_all_associations = 'no' ### YES only for protein prediction analysis
buildUCSCAnnoationFiles(species,mRNA_Type,export_all_associations,run_from_scratch,force)
reannotate_exon_seq = 'yes'
print 'genomic_build', genomic_build
if genomic_build == 'new':
### Need to run with every new genomic build (match up new coordinates
print "Begining to derive exon sequence from new genomic build"
JunctionArray.identifyCriticalExonLocations(species,array_type)
reannotate_exon_seq = 'yes'
JunctionArrayEnsemblRules.getAnnotations(species,array_type,reannotate_exon_seq,force)
### Download files required during AltAnalyze analysis but not during the database build process
filename = "AltDatabase/"+species+"/"+array_type+"/"+"MASTER-probeset-transcript.txt"
verifyFile(filename,array_type) ### Will force download if missing
filename = "AltDatabase/"+species+'/'+ array_type+'/'+array_type+"_annotations.txt"
verifyFile(filename,array_type) ### Will force download if missing
def import_annotations(filename, array_type):
import ExonAnalyze_module
fn = filepath(filename)
annotate_db = {}
x = 0
if array_type == 'AltMouse':
for line in open(fn, 'rU').xreadlines():
data = cleanUpLine(line)
if x == 0: x = 1
else:
try:
affygene, description, ll_id, symbol, rna_processing_annot = string.split(
data, '\t')
except ValueError:
affygene, description, ll_id, symbol = string.split(
data, '\t')
splicing_annotation = ''
if '"' in description:
null, description, null = string.split(description, '"')
rna_processing_annot = ''
y = ExonAnalyze_module.GeneAnnotationData(
affygene, description, symbol, ll_id, rna_processing_annot)
annotate_db[affygene] = y
else:
for line in open(fn, 'rU').xreadlines():
data = cleanUpLine(line)
if x == 0: x = 1
else:
rna_processing_annot = ''
try:
ensembl, description, symbol, rna_processing_annot = string.split(
data, '\t')
except ValueError:
ensembl, description, symbol = string.split(data, '\t')
y = ExonAnalyze_module.GeneAnnotationData(
ensembl, description, symbol, ensembl,
rna_processing_annot)
annotate_db[ensembl] = y
return annotate_db
def exportChromosomeStrandCoordinates(species):
import EnsemblImport
gene_location_db = EnsemblImport.getEnsemblGeneLocations(
species, 'RNASeq', 'key_by_array')
import ExpressionBuilder
gene_biotype_db = ExpressionBuilder.importTranscriptBiotypeAnnotations(
species)
export_path = 'GenMAPPDBs/' + species + '/chr_gene_locations.txt'
export_data = export.ExportFile(export_path)
import ExonAnalyze_module
gene_annotation_file = "AltDatabase/ensembl/" + species + "/" + species + "_Ensembl-annotations.txt"
annotate_db = ExonAnalyze_module.import_annotations(
gene_annotation_file, 'RNASeq')
print 'Annotations for', len(gene_location_db), 'genes imported'
sorted_list = []
protein_coding = 0
for gene in gene_location_db:
chr, strand, start, end = gene_location_db[gene]
if gene in gene_biotype_db:
biotype = gene_biotype_db[gene][-1]
if biotype == 'protein_coding': protein_coding += 1
else: biotype = 'NA'
if len(chr) < 7:
sorted_list.append(
[chr, strand, int(start),
int(end), gene, biotype])
#else: print chr;sys.exit()
print len(sorted_list), 'genes for typical chromosomes present'
print protein_coding, 'protein coding genes present'
sorted_list.sort()
for values in sorted_list:
chr, strand, start, end, gene, biotype = values
try:
symbol = annotate_db[gene].Symbol()
except Exception:
symbol = ''
values = [gene, symbol, chr, strand, str(start), str(end), biotype]
export_data.write(string.join(values, '\t') + '\n')
export_data.close()
print species, 'chromosome locations exported to:\n', export_path
def exportChromosomeStrandCoordinates(species):
import EnsemblImport
gene_location_db = EnsemblImport.getEnsemblGeneLocations(species,'RNASeq','key_by_array')
import ExpressionBuilder
gene_biotype_db = ExpressionBuilder.importTranscriptBiotypeAnnotations(species)
export_path = 'GenMAPPDBs/'+species+'/chr_gene_locations.txt'
export_data = export.ExportFile(export_path)
import ExonAnalyze_module
gene_annotation_file = "AltDatabase/ensembl/"+species+"/"+species+"_Ensembl-annotations.txt"
annotate_db = ExonAnalyze_module.import_annotations(gene_annotation_file,'RNASeq')
print 'Annotations for',len(gene_location_db),'genes imported'
sorted_list=[]; protein_coding=0
for gene in gene_location_db:
chr,strand,start,end = gene_location_db[gene]
if gene in gene_biotype_db:
biotype = gene_biotype_db[gene][-1]
if biotype == 'protein_coding': protein_coding+=1
else: biotype = 'NA'
if len(chr)<7:
sorted_list.append([chr,strand,int(start),int(end),gene,biotype])
#else: print chr;sys.exit()
print len(sorted_list),'genes for typical chromosomes present'
print protein_coding, 'protein coding genes present'
sorted_list.sort()
for values in sorted_list:
chr,strand,start,end,gene,biotype=values
try: symbol = annotate_db[gene].Symbol()
except Exception: symbol = ''
values = [gene,symbol,chr,strand,str(start),str(end),biotype]
export_data.write(string.join(values,'\t')+'\n')
export_data.close()
print species, 'chromosome locations exported to:\n',export_path
def displayExpressionGraph(species,
Platform,
exp_file,
gene,
transpose,
display=True,
showIntrons=False,
analysisType='plot'):
### Get gene annotations (users can provide an Ensembl or symbol)
print 'Importing exon-level expression data for visualization (be patient)...'
import ExonAnalyze_module
global platform
platform = Platform
if platform != 'AltMouse':
gene_annotation_file = "AltDatabase/ensembl/" + species + "/" + species + "_Ensembl-annotations.txt"
else:
gene_annotation_file = "AltDatabase/" + species + "/" + platform + "/" + platform + "_gene_annotations.txt"
genes = []
gene = string.replace(gene, '|', ',')
gene = string.replace(gene, ' ', ',')
if ',' in gene:
genes += string.split(gene, ',')
else:
genes.append(gene)
gene_db = {}
for gene in genes:
try:
if 'ENS' in gene:
try:
annotate_db ### If variable is defined
except Exception:
annotate_db = ExonAnalyze_module.import_annotations(
gene_annotation_file, platform,
keyBySymbol=False) ### Make an SQLite call
gene_symbol = annotate_db[gene].Symbol()
else:
try:
annotate_db ### If variable is defined
except Exception:
annotate_db = ExonAnalyze_module.import_annotations(
gene_annotation_file, platform, keyBySymbol=True)
gene_symbol = gene
gene = annotate_db[gene].GeneID()
gene_db[gene] = gene_symbol
except Exception:
print gene, 'not in database'
if len(gene_db) == 0:
force_no_gene_found_error
if 'AltResults' in exp_file:
root_dir = string.split(exp_file, 'AltResults')[0] + 'ExonPlots/'
else:
root_dir = string.split(exp_file, 'ExpressionInput')[0] + 'ExonPlots/'
import ExonAnalyze_module
if platform == 'RNASeq': datatype = 'exons'
else: datatype = 'probesets'
export_exon_filename = 'AltDatabase/' + species + '/' + platform + '/' + species + '_Ensembl_' + datatype + '.txt'
if verifyFileLength(export_exon_filename) == 0:
rootdir = string.replace(root_dir, 'ExonPlots/', '')
export_exon_filename = rootdir + '/' + export_exon_filename
import ExonArrayEnsemblRules
ensembl_exon_db = ExonArrayEnsemblRules.reimportEnsemblProbesetsForSeqExtraction(
export_exon_filename, 'gene-probesets',
gene_db) ### Make an SQLite call
filter_db = {}
for gene in ensembl_exon_db:
ensembl_exon_db[gene].sort()
for (index, ed, id) in ensembl_exon_db[gene]:
filter_db[id] = []
try:
os.mkdir(root_dir)
except Exception:
None ### dir exists
print 'Image results being saved to the folder "ExonPlots" in the AltAnalyze results directory.'
importTableEntries(exp_file,
filter_db,
ensembl_exon_db,
gene_db,
root_dir,
transpose,
display,
showIntrons,
analysisType=analysisType) ### Make an SQLite call
def displayExpressionGraph(species,Platform,exp_file,gene,transpose,display=True,showIntrons=False,analysisType='plot'):
### Get gene annotations (users can provide an Ensembl or symbol)
print 'Importing exon-level expression data for visualization (be patient)...'
import ExonAnalyze_module
global platform
platform = Platform
if platform != 'AltMouse': gene_annotation_file = "AltDatabase/ensembl/"+species+"/"+species+"_Ensembl-annotations.txt"
else: gene_annotation_file = "AltDatabase/"+species+"/"+platform+"/"+platform+"_gene_annotations.txt"
genes=[]
gene=string.replace(gene,'|',',')
gene=string.replace(gene,' ',',')
if ',' in gene:
genes += string.split(gene,',')
else: genes.append(gene)
gene_db={}
for gene in genes:
try:
if 'ENS' in gene:
try: annotate_db ### If variable is defined
except Exception:
annotate_db = ExonAnalyze_module.import_annotations(gene_annotation_file,platform,keyBySymbol=False) ### Make an SQLite call
gene_symbol = annotate_db[gene].Symbol()
else:
try: annotate_db ### If variable is defined
except Exception:
annotate_db = ExonAnalyze_module.import_annotations(gene_annotation_file,platform,keyBySymbol=True)
gene_symbol = gene
gene = annotate_db[gene].GeneID()
gene_db[gene]=gene_symbol
except Exception:
#if len(gene)>0: print gene, 'not in database'
pass
if len(gene_db)==0:
force_no_gene_found_error
if 'AltResults' in exp_file:
root_dir = string.split(exp_file,'AltResults')[0]+'ExonPlots/'
else:
root_dir = string.split(exp_file,'ExpressionInput')[0]+'ExonPlots/'
import ExonAnalyze_module
if platform == 'RNASeq': datatype = 'exons'
else: datatype = 'probesets'
export_exon_filename = 'AltDatabase/'+species+'/'+platform+'/'+species+'_Ensembl_'+datatype+'.txt'
if verifyFileLength(export_exon_filename) == 0:
rootdir = string.replace(root_dir,'ExonPlots/','')
export_exon_filename = rootdir+'/'+export_exon_filename
import ExonArrayEnsemblRules
ensembl_exon_db = ExonArrayEnsemblRules.reimportEnsemblProbesetsForSeqExtraction(export_exon_filename,'gene-probesets',gene_db) ### Make an SQLite call
filter_db = {}
for gene in ensembl_exon_db:
ensembl_exon_db[gene].sort()
for (index,ed,id) in ensembl_exon_db[gene]:
filter_db[id] = []
try: os.mkdir(root_dir)
except Exception: None ### dir exists
print 'Image results being saved to the folder "ExonPlots" in the AltAnalyze results directory.'
importTableEntries(exp_file,filter_db,ensembl_exon_db,gene_db,root_dir,transpose,display,showIntrons,analysisType=analysisType) ### Make an SQLite call
def alignAllDomainsToTranscripts(species,platform):
""" This function is only run during the database build process to create files available for subsequent download.
This recapitulates several functions executed during the database build process but does so explicitely for each
isoform with the goal of obtained genomic coordinates of each protein feature post de novo sequence alignment.
This includes all Ensembl proteins, UCSC mRNAs and in silico translated RNAs """
### Import all transcript to gene associations for Ensembl and UCSC transcripts
global gene_transcript_db
gene_transcript_db={}
option = 'transcript'
print 'Importing transcript data into memory'
filename = 'AltDatabase/ensembl/'+species+'/'+species+'_Ensembl_transcript-annotations.txt'
importEnsExonStructureData(filename,option)
filename = 'AltDatabase/ucsc/'+species+'/'+species+'_UCSC_transcript_structure_mrna.txt'
try: importEnsExonStructureData(filename,option)
except Exception: None ### Not available for all species - needs to be built prior to transcript model creation
import FeatureAlignment
ucsc_transcripts={}
gene_db = {}
gene_transcript_db = FeatureAlignment.eliminateRedundant(gene_transcript_db)
for gene in gene_transcript_db:
for (ac,type) in gene_transcript_db[gene]:
if type != 'Ensembl':
ucsc_transcripts[ac]=[] ### Store all the untranslated UCSC mRNAs
gene_db[gene] = [gene] ### mimics the necessary structure for FeatureAlignment
### Identify untranslated Ensembl transcripts
print 'Importing Ensembl transcript to protein'
ens_transcript_protein_db = importEnsemblTranscriptAssociations(species)
### Import protein ID and protein sequence into a dictionary
#global protein_sequence_db
#protein_sequence_db = FeatureAlignment.remoteEnsemblProtSeqImport(species) ### All Ensembl protein sequences
"""This code imports all protein sequences (NCBI, Ensembl, in silico translated) associated with optimal isoform pairs,
however, not all isoforms analyzed in the database are here, hence, this should be considered a subset of in silico
translated Ensembl mRNAs, UCSC ,RNAs, and known analyzed UCSC proteins"""
#ucsc_transcripts={}
#ucsc_transcripts['BC065499']=[]
#ucsc_transcripts['AK309510']=[] ### in silico translated
#ens_transcript_protein_db={}
### Download or translate ANY AND ALL mRNAs considered by AltAnalyze via in silico translation
import IdentifyAltIsoforms
analysis_type = 'fetch_new' # analysis_type = 'fetch' ???
#IdentifyAltIsoforms.remoteTranslateRNAs(species,ucsc_transcripts,ens_transcript_protein_db,analysis_type)
### Derive all protein ID, domain and genomic coordinate data from Ensembl and UniProt
""" This data is available for Ensembl and UniProt isoforms but we re-derive the associations based on sequence for completeness """
### Get the domain sequences and genomic coordinates
"""
# for testing
gt = {}; y=0
for gene in gene_db:
if y < 20:
gt[gene] = gene_db[gene]
else: break
y+=1
"""
protein_ft_db,domain_gene_counts = FeatureAlignment.grab_exon_level_feature_calls(species,platform,gene_db)
import ExonAnalyze_module
seq_files, mRNA_protein_seq_db = IdentifyAltIsoforms.importProteinSequences(species,'getSequence') ### Import all available protein sequences (downloaded or in silico)
coordinate_type = 'genomic'; coordinate_type = 'protein'
ExonAnalyze_module.getFeatureIsoformGenomePositions(species,protein_ft_db,mRNA_protein_seq_db,gene_transcript_db,coordinate_type)
def buildAltMouseExonAnnotations(species, array_type, force, genomic_build):
"""Code required to:
1) Extract out Affymetrix provided exon sequence (probeset sequence extracted from "probeset_sequence_reversed.txt", derived
directly from the Affymetrix AltMouse probe annotation file), from the "SEQUENCE-transcript-dbase.txt" (built using
dump-chip1 .gff sequence and AltMerge-Peptide Informatics script "sequence_analysis_AltMouse_refseq.py").
2) Once exported, grab full length exon sequences using exon/intron coordinates matches to full-length gene sequences with 2kb
flanking sequence to efficiently predict microRNA binding site exclusion (reAnnotateCriticalExonSequences) and later for
coordinate mapping to get exons aligning with UCSC annotated splicing annotations and exons. This sequence data replaced
the previous file (don't need to re-run this - see rederive_exonseq == 'yes' below for reference).
3) Match the updated exon sequences to the most recent genomic coordinates and build the exact equivalent of the exon array
Mm_Ensembl_probeset.txt database (same structure and ExonArrayEnsemblRules.py code). This involves running EnsemblImport.
This code should be run before the exon array location build code since the "Mm_Ensembl_probeset.txt" is created and then re-
written as "Mm_AltMouse_Ensembl_probeset.txt".
"""
import JunctionArray
import JunctionArrayEnsemblRules
rederive_exonseq = 'no'
### Only needs to be run once, to export exon sequence for AltMouse array the original (1 and 2 above)
if rederive_exonseq == 'yes':
import AltAnalyze
import ExonAnnotate_module
import ExonAnalyze_module
agglomerate_inclusion_probesets = 'no'
onlyAnalyzeJunctions = 'no'
probeset_annotations_file = "AltDatabase/" + species + "/" + array_type + "/" + "MASTER-probeset-transcript.txt"
verifyFile(probeset_annotations_file,
array_type) ### Will force download if missing
exon_db = {}
filtered_arrayids = {}
filter_status = 'no'
constituitive_probeset_db, exon_db, genes_being_analyzed = AltAnalyze.importSplicingAnnotationDatabase(
probeset_annotations_file, array_type, filtered_arrayids,
filter_status)
alt_junction_db, critical_exon_db, exon_dbase, exon_inclusion_db, exon_db = ExonAnnotate_module.identifyPutativeSpliceEvents(
exon_db, constituitive_probeset_db, {},
agglomerate_inclusion_probesets, onlyAnalyzeJunctions)
ExonAnnotate_module.exportJunctionComparisons(alt_junction_db,
critical_exon_db,
exon_dbase)
print "Finished exporting junctions used in AltMouse array comparisons."
ExonAnalyze_module.exportAltMouseExonSequence()
JunctionArray.reAnnotateCriticalExonSequences(species, array_type)
### Get UCSC associations (download databases if necessary)
mRNA_Type = 'mrna'
run_from_scratch = 'yes'
export_all_associations = 'no' ### YES only for protein prediction analysis
buildUCSCAnnoationFiles(species, mRNA_Type, export_all_associations,
run_from_scratch, force)
reannotate_exon_seq = 'yes'
print 'genomic_build', genomic_build
if genomic_build == 'new':
### Need to run with every new genomic build (match up new coordinates
print "Begining to derive exon sequence from new genomic build"
JunctionArray.identifyCriticalExonLocations(species, array_type)
reannotate_exon_seq = 'yes'
JunctionArrayEnsemblRules.getAnnotations(species, array_type,
reannotate_exon_seq, force)
### Download files required during AltAnalyze analysis but not during the database build process
filename = "AltDatabase/" + species + "/" + array_type + "/" + "MASTER-probeset-transcript.txt"
verifyFile(filename, array_type) ### Will force download if missing
filename = "AltDatabase/" + species + '/' + array_type + '/' + array_type + "_annotations.txt"
verifyFile(filename, array_type) ### Will force download if missing
def alignAllDomainsToTranscripts(species,platform):
""" This function is only run during the database build process to create files available for subsequent download.
This recapitulates several functions executed during the database build process but does so explicitely for each
isoform with the goal of obtained genomic coordinates of each protein feature post de novo sequence alignment.
This includes all Ensembl proteins, UCSC mRNAs and in silico translated RNAs """
### Import all transcript to gene associations for Ensembl and UCSC transcripts
global gene_transcript_db
gene_transcript_db={}
option = 'transcript'
print 'Importing transcript data into memory'
filename = 'AltDatabase/ensembl/'+species+'/'+species+'_Ensembl_transcript-annotations.txt'
importEnsExonStructureData(filename,option)
filename = 'AltDatabase/ucsc/'+species+'/'+species+'_UCSC_transcript_structure_mrna.txt'
try: importEnsExonStructureData(filename,option)
except Exception: None ### Not available for all species - needs to be built prior to transcript model creation
import FeatureAlignment
ucsc_transcripts={}
gene_db = {}
gene_transcript_db = FeatureAlignment.eliminateRedundant(gene_transcript_db)
for gene in gene_transcript_db:
for (ac,type) in gene_transcript_db[gene]:
if type != 'Ensembl':
ucsc_transcripts[ac]=[] ### Store all the untranslated UCSC mRNAs
gene_db[gene] = [gene] ### mimics the necessary structure for FeatureAlignment
### Identify untranslated Ensembl transcripts
print 'Importing Ensembl transcript to protein'
ens_transcript_protein_db = importEnsemblTranscriptAssociations(species)
### Import protein ID and protein sequence into a dictionary
#global protein_sequence_db
#protein_sequence_db = FeatureAlignment.remoteEnsemblProtSeqImport(species) ### All Ensembl protein sequences
"""This code imports all protein sequences (NCBI, Ensembl, in silico translated) associated with optimal isoform pairs,
however, not all isoforms analyzed in the database are here, hence, this should be considered a subset of in silico
translated Ensembl mRNAs, UCSC ,RNAs, and known analyzed UCSC proteins"""
#ucsc_transcripts={}
#ucsc_transcripts['BC065499']=[]
#ucsc_transcripts['AK309510']=[] ### in silico translated
#ens_transcript_protein_db={}
### Download or translate ANY AND ALL mRNAs considered by AltAnalyze via in silico translation
import IdentifyAltIsoforms
analysis_type = 'fetch_new' # analysis_type = 'fetch' ???
#IdentifyAltIsoforms.remoteTranslateRNAs(species,ucsc_transcripts,ens_transcript_protein_db,analysis_type)
### Derive all protein ID, domain and genomic coordinate data from Ensembl and UniProt
""" This data is available for Ensembl and UniProt isoforms but we re-derive the associations based on sequence for completeness """
### Get the domain sequences and genomic coordinates
"""
# for testing
gt = {}; y=0
for gene in gene_db:
if y < 20:
gt[gene] = gene_db[gene]
else: break
y+=1
"""
protein_ft_db,domain_gene_counts = FeatureAlignment.grab_exon_level_feature_calls(species,platform,gene_db)
import ExonAnalyze_module
seq_files, mRNA_protein_seq_db = IdentifyAltIsoforms.importProteinSequences(species,'getSequence') ### Import all available protein sequences (downloaded or in silico)
coordinate_type = 'genomic'; #coordinate_type = 'protein'
ExonAnalyze_module.getFeatureIsoformGenomePositions(species,protein_ft_db,mRNA_protein_seq_db,gene_transcript_db,coordinate_type)
