class TestGOModule(unittest.TestCase):
def setUp(self):
logging.basicConfig(filename=None, level="ERROR", format='%(asctime)s - %(name)s - %(levelname)s: %(message)s')
logger.info("Starting DataManager tests")
self.this_dir = os.path.split(__file__)[0]
self.conf_parser = GenedescConfigParser(os.path.join(self.this_dir, os.path.pardir, "tests", "config_test.yml"))
self.df = DataManager(do_relations=None, go_relations=["subClassOf", "BFO:0000050"])
logger.info("Loading go ontology from file")
self.df.load_ontology_from_file(ontology_type=DataType.GO, ontology_url="file://" + os.path.join(
self.this_dir, "data", "go_gd_test.obo"), ontology_cache_path=os.path.join(self.this_dir, "cache",
"go_gd_test.obo"),
config=self.conf_parser)
logger.info("Loading go associations from file")
self.df.load_associations_from_file(associations_type=DataType.GO, associations_url="file://" + os.path.join(
self.this_dir, "data", "gene_association_1.7.wb.partial"),
associations_cache_path=os.path.join(self.this_dir, "cache",
"gene_association_1.7.wb.partial"),
config=self.conf_parser)
def test_ontology_exists(self):
self.assertTrue(self.df.go_ontology is not None)
self.assertTrue(any(parent == "GO:0009987" for parent in
self.df.go_ontology.parents("GO:0000075")))
def test_annotations_exist(self):
self.assertTrue(self.df.go_associations is not None)
self.assertTrue(len(self.df.get_annotations_for_gene(
gene_id="WB:WBGene00000001", annot_type=DataType.GO,
include_obsolete=False, include_negative_results=False,
priority_list=self.conf_parser.get_annotations_priority(module=Module.GO))) > 0)
def test_rename_terms(self):
self.assertTrue(all(len(self.df.go_ontology.search(term)) == 0 for term in list(
self.conf_parser.get_module_property(module=Module.GO, prop=ConfigModuleProperty.RENAME_TERMS).keys())))
def test_exclude_terms(self):
test_annot = self.df.get_annotations_for_gene("WB:WBGene00000001", annot_type=DataType.GO)
self.assertTrue(all([annot["object"]["id"] != "GO:0008286" for annot in test_annot]))
def test_download_gz_file(self):
test_file = self.df._get_cached_file(cache_path=os.path.join(self.this_dir, "cache",
"c_elegans.PRJNA13758.WS273.geneIDs.txt.gz"),
file_source_url="file://" + os.path.join(
self.this_dir, "data", "c_elegans.PRJNA13758.WS273.geneIDs.txt.gz"))
self.assertTrue(test_file == os.path.join(self.this_dir, "cache", "c_elegans.PRJNA13758.WS273.geneIDs.txt"))
def test_gene_data_functions(self):
self.df.set_gene_data(gene_data=[Gene("1", "gene1", True, False), Gene("2", "gene2", False, True),
Gene("3", "gene3", False, False), Gene("4", "gene4", True, True)])
self.assertTrue(len([g for g in self.df.get_gene_data(include_dead_genes=False,
include_pseudo_genes=False)]) == 1)
self.assertTrue(len([g for g in self.df.get_gene_data(include_dead_genes=True,
include_pseudo_genes=False)]) == 2)
self.assertTrue(len([g for g in self.df.get_gene_data(include_dead_genes=False,
include_pseudo_genes=True)]) == 2)
self.assertTrue(len([g for g in self.df.get_gene_data(include_dead_genes=True,
include_pseudo_genes=True)]) == 4)
def test_get_human_gene_props(self):
human_gene_props = self.df.get_human_gene_props()
self.assertTrue(len(human_gene_props) > 0)
def test_get_ensembl_hgnc_ids_map(self):
ensembl_hgnc_ids_map = self.df.get_ensembl_hgnc_ids_map()
self.assertTrue(len(ensembl_hgnc_ids_map) > 0)
def test_set_ontology(self):
ontology = OntologyFactory().create()
for i in range(4):
ontology.add_node(i, 'node' + str(i))
ontology.add_parent(1, 0)
ontology.add_parent(2, 0)
ontology.add_parent(3, 0)
self.df.set_ontology(ontology_type=DataType.GO, ontology=ontology, config=self.conf_parser)
self.assertTrue(list(self.df.go_ontology.nodes()) == list(ontology.nodes()))
def test_set_associations(self):
associations = []
associations.append(DataManager.create_annotation_record("", "1", "a", "protein_coding", "001", "GO:0019901",
"", "F", "EXP", None, "WB", ""))
associations.append(DataManager.create_annotation_record("", "2", "b", "protein_coding", "001", "GO:0005515",
"", "F", "EXP", None, "WB", ""))
assocs = AssociationSetFactory().create_from_assocs(assocs=associations, ontology=self.df.go_ontology)
self.df.set_associations(associations_type=DataType.GO, associations=assocs, config=self.conf_parser)
self.assertTrue(self.df.go_associations)
def test_remap_associations(self):
associations = []
associations.append(DataManager.create_annotation_record("", "1", "a", "protein_coding", "001", "GO:0018996",
"", "F", "EXP", None, "WB", ""))
assocs = AssociationSetFactory().create_from_assocs(assocs=associations, ontology=self.df.go_ontology)
self.df.set_associations(associations_type=DataType.GO, associations=assocs, config=self.conf_parser)
self.assertEqual(self.df.go_associations.associations_by_subj["1"][0]["object"]["id"], "GO:0042303")
def _load_and_process_data(self):
# create gene descriptions data manager and load common data
context_info = ContextInfo()
data_manager = DataFileManager(context_info.config_file_location)
#go_onto_config = data_manager.get_config('GO')
go_annot_config = data_manager.get_config('GAF')
#do_onto_config = data_manager.get_config('DOID')
go_annot_sub_dict = {sub.get_data_provider(): sub for sub in go_annot_config.get_sub_type_objects()}
this_dir = os.path.split(__file__)[0]
gd_config = GenedescConfigParser(os.path.join(this_dir,
os.pardir,
os.pardir,
"gene_descriptions.yml"))
gd_data_manager = DataManager(do_relations=None, go_relations=["subClassOf", "BFO:0000050"])
gd_data_manager.set_ontology(ontology_type=DataType.GO,
ontology=self.get_ontology(data_type=DataType.GO),
config=gd_config)
gd_data_manager.set_ontology(ontology_type=DataType.DO,
ontology=self.get_ontology(data_type=DataType.DO),
config=gd_config)
# generate descriptions for each MOD
for prvdr in [sub_type.get_data_provider().upper() \
for sub_type in self.data_type_config.get_sub_type_objects()]:
gd_config_mod_specific = copy.deepcopy(gd_config)
if prvdr == "WB":
gd_config_mod_specific.config["expression_sentences_options"][
"remove_children_if_parent_is_present"] = True
self.logger.info("Generating gene descriptions for %s", prvdr)
data_provider = prvdr if prvdr != "HUMAN" else "RGD"
json_desc_writer = DescriptionsWriter()
go_annot_path = "file://" + os.path.join(os.getcwd(),
"tmp",
go_annot_sub_dict[prvdr].file_to_download)
gd_data_manager.load_associations_from_file(
associations_type=DataType.GO, associations_url=go_annot_path,
associations_cache_path=os.path.join(os.getcwd(),
"tmp",
"gd_cache",
"go_annot_" + prvdr + ".gaf"),
config=gd_config_mod_specific)
gd_data_manager.set_associations(associations_type=DataType.DO,
associations=self.get_disease_annotations_from_db(
data_provider=data_provider,
gd_data_manager=gd_data_manager,
logger=self.logger),
config=gd_config_mod_specific)
if prvdr in EXPRESSION_PRVD_SUBTYPE_MAP:
gd_data_manager.set_ontology(ontology_type=DataType.EXPR,
ontology=self.get_ontology(data_type=DataType.EXPR,
provider=prvdr),
config=gd_config_mod_specific)
gd_data_manager.set_associations(
associations_type=DataType.EXPR,
associations=self.get_expression_annotations_from_db(data_provider=data_provider,
gd_data_manager=gd_data_manager,
logger=self.logger),
config=gd_config_mod_specific)
commit_size = self.data_type_config.get_neo4j_commit_size()
generators = self.get_generators(prvdr,
gd_data_manager,
gd_config_mod_specific,
json_desc_writer)
query_template_list = [
[self.gene_descriptions_query_template, commit_size,
"genedescriptions_data_" + prvdr + ".csv"]
]
query_and_file_list = self.process_query_params(query_template_list)
CSVTransactor.save_file_static(generators, query_and_file_list)
Neo4jTransactor.execute_query_batch(query_and_file_list)
self.save_descriptions_report_files(data_provider=prvdr,
json_desc_writer=json_desc_writer,
context_info=context_info,
gd_data_manager=gd_data_manager)