def test_gene_sets(self):
gs1 = GeneSet(
gs_id=self.test_gs_id,
name=self.test_name,
genes=self.test_genes,
hierarchy=self.test_hierarchy,
organism=self.test_organism,
link='',
)
gs2 = GeneSet(gs_id='test2',
name='test_name2',
hierarchy=('Test', 'test'),
organism='3702')
gs3 = GeneSet(gs_id='test3',
name='test_name3',
hierarchy=('Test', 'test'),
organism='3702')
sets = GeneSets([gs1, gs2, gs3])
self.assertIsNotNone(sets)
self.assertRaises(GeneSetException, sets.common_org)
self.assertRaises(GeneSetException, sets.common_hierarchy)
self.assertGreater(len(sets.hierarchies()), 1)
split_by_hierarchy = sets.split_by_hierarchy()
self.assertLess(len(split_by_hierarchy), len(sets))
def test_gene_set(self):
gs1 = GeneSet(gs_id=self.test_gs_id,
name=self.test_name,
genes=self.test_genes,
hierarchy=self.test_hierarchy,
organism=self.test_organism,
link='')
gs2 = GeneSet(gs_id='test2', name='test_name2')
self.assertEqual(gs1.gmt_description(), 'test_gs,GO-biological_process,9606,test_name,_,_,_')
self.assertFalse(gs1 == gs2)
self.assertNotEqual(gs1, gs2)
self.assertTrue(gs1 == gs1)
def dicty_mutant_gene_sets(org):
""" Return dicty mutant phenotype gene sets from Dictybase
"""
if org == '352472':
gene_sets = []
gene_matcher = GeneMatcher('352472')
for phenotype, mutants in dicty.phenotypes.phenotype_mutants().items():
gene_symbols = [
dicty.phenotypes.mutant_genes(mutant)[0] for mutant in mutants
]
gene_matcher.genes = gene_symbols
gene_matcher.run_matcher()
genes = []
for gene in gene_matcher.genes:
if gene.ncbi_id is not None:
genes.append(int(gene.ncbi_id))
if len(gene_symbols) != len(genes):
print(len(gene_symbols), len(genes))
gs = GeneSet(gs_id=phenotype,
name=phenotype,
genes=genes,
hierarchy=('Dictybase', 'Phenotypes'),
organism='352472',
link='')
gene_sets.append(gs)
return GeneSets(gene_sets)
def kegg_gene_sets(org):
""" Returns gene sets from KEGG pathways.
"""
caching.clear_cache()
kegg_org = kegg.KEGGOrganism(taxonomy.name(org))
ncbi_id_mapper = kegg_org.kegg_to_ncbi_mapper()
genesets = []
for id in kegg_org.pathways():
pway = kegg.KEGGPathway(id)
hier = ('KEGG', 'pathways')
if pway.pathway_attributes():
kegg_names = kegg_org.get_genes_by_pathway(id)
mapped_genes = []
for gene in kegg_names:
try:
mapped_genes.append(ncbi_id_mapper[gene.upper()])
except KeyError:
# some kegg names can not be matched to ncbi ids
# they are included in geneset anyway
# remove prefix, that specifies kegg organism
# mapped_genes.append(gene.split(':')[-1])
pass
gs = GeneSet(gs_id=id,
name=pway.title,
genes=mapped_genes,
hierarchy=hier,
organism=org,
link=pway.link)
genesets.append(gs)
return GeneSets(genesets)
def omim_gene_sets(org):
""" Return gene sets from OMIM (Online Mendelian Inheritance in Man) diseses
"""
if org == '9606':
gene_matcher = GeneMatcher('9606')
genesets = []
for disease in omim.diseases():
gene_symbols = omim.disease_genes(disease)
gene_matcher.genes = gene_symbols
gene_matcher.run_matcher()
genes = []
for gene in gene_matcher.genes:
if gene.ncbi_id is not None:
genes.append(int(gene.ncbi_id))
gs = GeneSet(
gs_id=disease.id,
name=disease.name,
genes=genes,
hierarchy=('OMIM', ),
organism='9606',
link=(OMIM_LINK.format(disease.id) if disease.id else None))
genesets.append(gs)
return GeneSets(genesets)
def reactome_gene_sets(org):
""" Prepare human pathways gene sets from reactome pathways
"""
if org == '9606':
gene_matcher = GeneMatcher('9606')
with urlopen(REACTOME_DOWNLOAD_LINK) as url:
memfile = io.BytesIO(url.read())
with ZipFile(memfile, 'r') as myzip:
f = myzip.open(REACTOME_FILE_NAME)
content = f.read().decode().splitlines()
genesets = []
for path in content:
gene_symbols = path.split('\t')[2:] if path.split(
'\t')[2:] else []
gene_matcher.genes = gene_symbols
gene_matcher.run_matcher()
genes = []
for gene in gene_matcher.genes:
if gene.ncbi_id is not None:
genes.append(int(gene.ncbi_id))
gs = GeneSet(gs_id=path.split('\t')[0],
name=path.split('\t')[0],
genes=genes,
hierarchy=('Reactome', 'Pathways'),
organism='9606',
link='')
genesets.append(gs)
return GeneSets(genesets)
def cytoband_gene_sets(org):
""" Create cytoband gene sets from Stanford Microarray Database
"""
if org == '9606':
gene_matcher = GeneMatcher('9606')
with urlopen(CYTOBAND_DOWNLOAD_LINK) as stream:
data = stream.read().splitlines()
genesets = []
for band in data:
b = band.decode().split('\t')
gene_symbols = b[2:]
gene_matcher.genes = gene_symbols
gene_matcher.run_matcher()
genes = []
for gene in gene_matcher.genes:
if gene.ncbi_id is not None:
genes.append(int(gene.ncbi_id))
genesets.append(
GeneSet(gs_id=b[0],
name=b[1],
genes=genes if b[2:] else [],
hierarchy=('Cytobands', ),
organism='9606',
link=''))
return GeneSets(genesets)
def add_custom_sets(self,
gene_sets_names,
gene_names,
hierarchy_title=None,
select_customs_flag=False):
# type: (np.ndarray, np.ndarray) -> None
self.custom_set_hier = hierarchy_title
self.clear_custom_sets()
temp_dict = defaultdict(list)
for set_name, gene_name in zip(gene_sets_names, gene_names):
temp_dict[set_name].append(gene_name)
g_sets = []
for key, value in temp_dict.items():
g_sets.append(
GeneSet(gs_id=key,
hierarchy=self.custom_set_hier,
organism=self.gs_object.common_org(),
name=key,
genes=set(value)))
self.gs_object.update(g_sets)
self.update_gs_hierarchy(select_customs_flag=select_customs_flag)
def test_gmt_file_format(self):
gs = GeneSet(
gs_id=self.test_gs_id,
name=self.test_name,
genes=self.test_genes,
hierarchy=self.test_hierarchy,
organism=self.test_organism,
link='',
)
fd, file_name = mkstemp()
# write to file
write_sets = GeneSets([gs])
write_sets.to_gmt_file_format(file_name)
with open(file_name, 'r') as temp_f:
line = temp_f.readline()
columns = line.strip().split('\t')
self.assertGreater(len(columns), 0)
# read from file
read_sets = GeneSets.from_gmt_file_format(file_name)
self.assertIsNotNone(read_sets)
self.assertGreater(len(read_sets), 0)
self.assertEqual(read_sets.common_hierarchy(), self.test_hierarchy)
self.assertEqual(read_sets.common_org(), self.test_organism)
# clean-up
os.close(fd)
os.remove(file_name)
def dicty_mutant_gene_sets(tax_id: str):
""" Return dicty mutant phenotype gene sets from Dictybase
"""
if tax_id == '44689':
gene_sets = []
gene_matcher = GeneMatcher('44689')
for phenotype, mutants in phenotypes.phenotype_mutants().items():
phenotype = phenotype.replace(",", " ")
gene_symbols = [
phenotypes.mutant_genes(mutant)[0] for mutant in mutants
]
gene_matcher.genes = gene_symbols
genes = set()
for gene in gene_matcher.genes:
if gene.gene_id is not None:
genes.add(str(gene.gene_id))
gs = GeneSet(gs_id=phenotype,
name=phenotype,
genes=genes,
hierarchy=('Dictybase', 'Phenotypes'),
organism=tax_id,
link='')
gene_sets.append(gs)
for gs_group in GeneSets(gene_sets).split_by_hierarchy():
hierarchy = gs_group.common_hierarchy()
gs_group.to_gmt_file_format(
f'{data_path}/gene_sets/{filename(hierarchy, tax_id)}')
def cytoband_gene_sets(tax_id: str) -> None:
""" Create cytoband gene sets from Stanford Microarray Database
"""
if tax_id == '9606':
download_link = 'http://statweb.stanford.edu/~tibs/GSA/cytobands-stanford.gmt'
gene_matcher = GeneMatcher('9606')
with urlopen(download_link) as stream:
data = stream.read().splitlines()
genesets = []
for band in data:
b = band.decode().split('\t')
gene_symbols = b[2:]
gene_matcher.genes = gene_symbols
genes = set()
for gene in gene_matcher.genes:
if gene.gene_id is not None:
genes.add(gene.gene_id)
genesets.append(
GeneSet(gs_id=b[0],
name=b[1],
genes=genes if b[2:] else set(),
hierarchy=('Cytobands', ),
organism='9606',
link=''))
for gs_group in GeneSets(genesets).split_by_hierarchy():
hierarchy = gs_group.common_hierarchy()
gs_group.to_gmt_file_format(
f'{data_path}/gene_sets/{filename(hierarchy, tax_id)}')
def to_gene_set(term: go.Term) -> Optional[GeneSet]:
genes = annotations.get_genes_by_go_term(term.id)
if len(genes) > 0:
return GeneSet(
gs_id=term.id,
name=term.name,
genes=set(genes),
hierarchy=('GO', term.namespace),
organism=tax_id,
link=f'http://amigo.geneontology.org/amigo/term/{term.id}')
def reactome_gene_sets(tax_id: str) -> None:
""" Prepare human pathways gene sets from reactome pathways
"""
if tax_id == '9606':
download_link = 'http://www.reactome.org/download/current/ReactomePathways.gmt.zip'
file_name = 'ReactomePathways.gmt'
detail_link = 'https://reactome.org/content/detail/{}'
gene_matcher = GeneMatcher('9606')
with urlopen(download_link) as url:
memfile = io.BytesIO(url.read())
with ZipFile(memfile, 'r') as myzip:
f = myzip.open(file_name)
content = f.read().decode().splitlines()
genesets = []
for path in content:
gene_symbols = path.split('\t')[2:] if path.split(
'\t')[2:] else []
gene_matcher.genes = gene_symbols
genes = set()
for gene in gene_matcher.genes:
if gene.gene_id is not None:
genes.add(str(gene.gene_id))
pathway = path.split('\t')[0].replace(',', ' ')
pathway_id = path.split('\t')[1].replace(',', ' ')
gs = GeneSet(gs_id=pathway_id,
name=pathway,
genes=genes,
hierarchy=('Reactome', 'pathways'),
organism='9606',
link=detail_link.format(pathway_id))
genesets.append(gs)
for gs_group in GeneSets(genesets).split_by_hierarchy():
hierarchy = gs_group.common_hierarchy()
gs_group.to_gmt_file_format(
f'{data_path}/gene_sets/{filename(hierarchy, tax_id)}')
def go_gene_sets(org):
""" Returns gene sets from GO.
"""
ontology = go.Ontology()
annotations = go.Annotations(org, ontology=ontology)
gene_sets = []
for termn, term in ontology.terms.items():
genes = annotations.get_genes_by_go_term(termn)
hier = ('GO', term.namespace)
if len(genes) > 0:
gs = GeneSet(gs_id=termn, name=term.name, genes=genes, hierarchy=hier,
organism=org, link=GO_TERM_LINK.format(termn))
gene_sets.append(gs)
return GeneSets(gene_sets)
def kegg_gene_sets(tax_id: str) -> None:
""" Returns gene sets from KEGG pathways.
"""
caching.clear_cache()
kegg_org = kegg.KEGGOrganism(taxonomy.name(tax_id))
ncbi_id_mapper = kegg_org.kegg_to_ncbi_mapper()
genesets = []
for id in kegg_org.pathways():
pway = kegg.KEGGPathway(id)
hier = ('KEGG', 'Pathways')
if pway.pathway_attributes():
kegg_names = kegg_org.get_genes_by_pathway(id)
mapped_genes = set()
for gene in kegg_names:
try:
mapped_genes.add(ncbi_id_mapper[gene.upper()])
except KeyError:
# some kegg names can not be matched to ncbi ids
# they are included in geneset anyway
# remove prefix, that specifies kegg organism
# mapped_genes.append(gene.split(':')[-1])
pass
gs = GeneSet(gs_id=id,
name=pway.title,
genes=mapped_genes,
hierarchy=hier,
organism=tax_id,
link=pway.link)
genesets.append(gs)
for gs_group in GeneSets(genesets).split_by_hierarchy():
hierarchy = gs_group.common_hierarchy()
gs_group.to_gmt_file_format(
f'{data_path}/gene_sets/{filename(hierarchy, tax_id)}')
def gene_marker_sets():
file_names = ['panglao_gene_markers.tab', 'cellMarker_gene_markers.tab']
file_name_to_hier = {
'panglao_gene_markers.tab': 'Panglao',
'cellMarker_gene_markers.tab': 'CellMarker'
}
for file_name in file_names:
file_path = f'{data_path}/marker_genes/{file_name}'
sets_by_org = {'9606': defaultdict(list), '10090': defaultdict(list)}
name_to_tax = {'Human': '9606', 'Mouse': '10090'}
for row in Table(file_path):
tax_id = name_to_tax[row['Organism']]
cell_type = row['Cell Type']
gene_id = row['Entrez ID']
sets_by_org[tax_id][cell_type].append(gene_id)
for tax_id, cell_types in sets_by_org.items():
gene_sets = []
for cell_type, genes in cell_types.items():
gs = GeneSet(
gs_id=str(cell_type),
name=str(cell_type),
genes=set([str(gene) for gene in genes if gene != '?']),
hierarchy=('Marker Genes', file_name_to_hier[file_name]),
organism=tax_id,
link='')
gene_sets.append(gs)
for gs_group in GeneSets(gene_sets).split_by_hierarchy():
hierarchy = gs_group.common_hierarchy()
gs_group.to_gmt_file_format(
f'{data_path}/gene_sets/{filename(hierarchy, tax_id)}')