def ensembl_gene_parse(cls, *args, **kwargs):
''' Parse gene GTF file from ensembl. '''
stage_file = cls._get_stage_file(*args, **kwargs)
download_file = cls._get_download_file(*args, **kwargs)
Gene.gene_mapping(kwargs['section']['index'], kwargs['section']['index_type'])
with gzip.open(download_file, 'rt') as ensembl_gene_f:
with open(stage_file, 'w') as outfile:
json.dump(Gene.ensembl_gene_parse(ensembl_gene_f), outfile, indent=0)
def test_gene_pipeline(self):
""" Test gene pipeline. """
INI_CONFIG = IniParser().read_ini(MY_INI_FILE)
idx = INI_CONFIG["ENSEMBL_GENE_GTF"]["index"]
idx_type = INI_CONFIG["ENSEMBL_GENE_GTF"]["index_type"]
""" 1. Test ensembl GTF loading. """
call_command(
"pipeline", "--steps", "stage", "load", sections="ENSEMBL_GENE_GTF", dir=TEST_DATA_DIR, ini=MY_INI_FILE
)
Search.index_refresh(idx)
elastic = Search(idx=idx, idx_type=idx_type)
self.assertGreaterEqual(elastic.get_count()["count"], 1, "Count documents in the index")
map1_props = Gene.gene_mapping(idx, idx_type, test_mode=True).mapping_properties
map2_props = elastic.get_mapping()
if idx not in map2_props:
logger.error("MAPPING ERROR: " + json.dumps(map2_props))
self._cmpMappings(map2_props[idx]["mappings"], map1_props, idx_type)
""" 2. Test adding entrez ID to documents """
call_command("pipeline", "--steps", "load", sections="GENE2ENSEMBL", dir=TEST_DATA_DIR, ini=MY_INI_FILE)
Search.index_refresh(idx)
query = ElasticQuery.query_string("PTPN22", fields=["symbol"])
elastic = Search(query, idx=idx)
docs = elastic.search().docs
self.assertEqual(len(docs), 1)
self.assertTrue("entrez" in getattr(docs[0], "dbxrefs"))
self.assertEqual(getattr(docs[0], "dbxrefs")["entrez"], "26191")
""" 3. Add uniprot and fill in missing entrez fields. """
call_command(
"pipeline", "--steps", "download", "load", sections="ENSMART_GENE", dir=TEST_DATA_DIR, ini=MY_INI_FILE
)
Search.index_refresh(idx)
query = ElasticQuery.query_string("DNMT3L", fields=["symbol"])
elastic = Search(query, idx=idx)
docs = elastic.search().docs
self.assertTrue("entrez" in getattr(docs[0], "dbxrefs"))
self.assertTrue("swissprot" in getattr(docs[0], "dbxrefs"))
""" 4. Add gene synonyms and dbxrefs. """
call_command("pipeline", "--steps", "load", sections="GENE_INFO", dir=TEST_DATA_DIR, ini=MY_INI_FILE)
Search.index_refresh(idx)
query = ElasticQuery.query_string("PTPN22", fields=["symbol"])
elastic = Search(query, idx=idx)
docs = elastic.search().docs
self.assertTrue("PTPN8" in getattr(docs[0], "synonyms"))
""" 5. Add PMIDs to gene docs. """
call_command("pipeline", "--steps", "load", sections="GENE_PUBS", dir=TEST_DATA_DIR, ini=MY_INI_FILE)
Search.index_refresh(idx)
query = ElasticQuery.query_string("PTPN22", fields=["symbol"])
elastic = Search(query, idx=idx)
docs = elastic.search().docs
self.assertGreater(len(getattr(docs[0], "pmids")), 0)
""" 6. Add ortholog data. """
call_command("pipeline", "--steps", "load", sections="ENSMART_HOMOLOG", dir=TEST_DATA_DIR, ini=MY_INI_FILE)
Search.index_refresh(idx)
query = ElasticQuery.query_string("PTPN22", fields=["symbol"])
elastic = Search(query, idx=idx)
docs = elastic.search().docs
dbxrefs = getattr(docs[0], "dbxrefs")
self.assertTrue("orthologs" in dbxrefs, dbxrefs)
self.assertTrue("mmusculus" in dbxrefs["orthologs"], dbxrefs)
self.assertEqual("ENSMUSG00000027843", dbxrefs["orthologs"]["mmusculus"]["ensembl"])
query = ElasticQuery.filtered(
Query.match_all(),
TermsFilter.get_terms_filter("dbxrefs.orthologs.mmusculus.ensembl", ["ENSMUSG00000027843"]),
)
docs = Search(query, idx=idx, size=1).search().docs
self.assertEqual(len(docs), 1)
""" 7. Add mouse ortholog link to MGI """
call_command("pipeline", "--steps", "load", sections="ENSEMBL2MGI", dir=TEST_DATA_DIR, ini=MY_INI_FILE)
Search.index_refresh(idx)
docs = Search(query, idx=idx, size=1).search().docs
dbxrefs = getattr(docs[0], "dbxrefs")
self.assertEqual("ENSMUSG00000027843", dbxrefs["orthologs"]["mmusculus"]["ensembl"])
self.assertEqual("107170", dbxrefs["orthologs"]["mmusculus"]["MGI"])
