def _logically_verifies_cleaned_ontologies(self) -> None:
"""Logically verifies an ontology by running the ELK deductive logic reasoner. Before running the reasoner
the instantiated RDFLib object is saved locally.
Returns:
None.
"""
log_str = 'Logically Verifying Ontology'
print(log_str)
logger.info('PKT: ' + log_str)
# save graph in order to run reasoner
filename = self.temp_dir + '/' + self.ont_file_location
self.ont_graph.serialize(destination=filename, format='xml')
command = "{} {} --reasoner elk --run-reasoner --assert-implied -o {}"
return_code = os.system(
command.format(self.owltools_location, filename, filename))
if return_code == 0:
if isinstance(self.bucket, storage.bucket.Bucket):
uploads_data_to_gcs_bucket(self.bucket, self.processed_data,
self.temp_dir,
self.ont_file_location)
else:
log_str = 'ERROR: Reasoner Finished with Errors - {}: {}'.format(
filename, return_code)
logger.error(log_str)
raise Exception(log_str)
return None
def _run(self):
"""Method uploads any log files found in temp_directory from a local directory to a specific directory in a
Google Cloud Storage Bucket every "n" minutes as specified by the input interval variable. This method runs
the program it is called with finishes. There is also a back-up timer that will kill the program
Args:
bucket: A storage Bucket object specifying a Google Cloud Storage bucket.
original_data: A string specifying the location of the original_data directory for a specific build.
temp_directory: A local directory where preprocessed data is stored.
interval: An integer specifying how often the data should be pushed up to the Google Cloud Storage Bucket.
Returns:
None.
"""
# grep for log files in the log_directory
log_file = glob.glob(self.log_directory + '/*.log')[0].split('/')[-1]
runtime = 0
while runtime < self.kill_time:
uploads_data_to_gcs_bucket(self.bucket, self.gcs_bucket_location,
self.log_directory, log_file)
time.sleep(self.sleep)
runtime += self.sleep
return None
def main():
start_time = datetime.now()
# initialize Google Cloud Storage Bucket object and delete prior logs (if present) from current_build directory
bucket = storage.Client().get_bucket('pheknowlator'); gcs_log_loc = 'temp_build_inprogress/'
try: deletes_single_file(bucket, gcs_log_loc + '{}'.format(log))
except NotFound: pass
# start logger and configure Google Cloud Storage settings
log_str = ' STARTING PHEKNOWLATOR KNOWLEDGE GRAPH BUILD '
print('\n\n' + '*' * 10 + log_str + '*' * 10); logger.info('*' * 10 + log_str + '*' * 10)
uploads_data_to_gcs_bucket(bucket, gcs_log_loc, log_dir, log)
# run phase 1 of build
log_str = 'BUILD PHASE 1: DOWNLOADING BUILD DATA'
print('#' * 35 + '\n' + log_str + '\n' + '#' * 35); logger.info('#' * 5 + log_str + '#' * 5)
try: run_phase_1()
except: logger.error('ERROR: Uncaught Exception: {}'.format(traceback.format_exc()))
uploads_data_to_gcs_bucket(bucket, gcs_log_loc, log_dir, log)
# run phase 2 build
log_str = 'BUILD PHASE 2: DATA PRE-PROCESSING'
print('#' * 35 + '\n' + log_str + '\n' + '#' * 35); logger.info('#' * 5 + log_str + '#' * 5)
try: run_phase_2()
except: logger.error('ERROR: Uncaught Exception: {}'.format(traceback.format_exc()))
uploads_data_to_gcs_bucket(bucket, gcs_log_loc, log_dir, log)
# print build statistics and upload logging for data preprocessing and ontology cleaning
runtime = round((datetime.now() - start_time).total_seconds() / 60, 3)
log_str = ' COMPLETED BUILD PHASES 1-2: {} MINUTES '.format(runtime)
print('\n\n' + '*' * 10 + log_str + '*' * 10); logger.info(log_str) # don't delete needed for build monitoring
logger.info('EXIT BUILD PHASES 1-2') # don't delete needed for build monitoring
uploads_data_to_gcs_bucket(bucket, gcs_log_loc, log_dir, log)
def cleans_ontology_data(self) -> None:
"""Performs all needed ontology cleaning tasks by resolving different types of ontology cleaning steps at the
individual ontology- and the merged ontology-level, each are described below:
- Individual Ontologies: (1) Parsing Errors, (2) Identifier Errors, (3) Deprecated/Obsolete Errors, and (4)
Punning Errors.
- Merged Ontologies: (1) Identifier Errors, (2) Normalizes Duplicate and Existing Concepts, and (3) Punning
Errors.
NOTE. The OWL API, when running the ELK reasoner, seems to add back some of the errors that this script removes.
Example 1: In the Vaccine Ontology, we fix prefix errors where "PR" is recorded as "PRO". If you save the
ontology without running the reasoner and reload it, the fix remains.
Example 2: When we create the human subset of the Protein Ontology we verify that it contains only a
single large connected component.
For both examples, if you run the reasoner ELK, save the ontology with inferences, and re-load it, "PRO" will
re-appear and the human pro ontology with contain 3 connected components. Luckily, the merged ontologies are
not reasoned, thus the version used to build knowledge graphs is free of these errors.
Returns:
None.
"""
log_str = '*** CLEANING INDIVIDUAL ONTOLOGY DATA SOURCES ***'
print(log_str)
logger.info(log_str)
for ont in self.ontology_info.keys():
if ont != self.merged_ontology_filename:
print('\nProcessing Ontology: {}'.format(ont.upper()))
logger.info('\nProcessing Ontology: {}'.format(ont.upper()))
self.ont_file_location, self.ont_graph = ont, self.reads_gcs_bucket_data_to_graph(
ont)
self.updates_ontology_reporter() # get starting statistics
self.fixes_ontology_parsing_errors()
self.fixes_identifier_errors()
self.removes_deprecated_obsolete_entities()
self.fixes_punning_errors()
self._logically_verifies_cleaned_ontologies()
# read in cleaned, verified, and updated ontology containing inference
log_str = 'Reading in Cleaned Ontology -- Needed to Calculate Final Statistics'
print(log_str)
logger.info(log_str)
self.ont_graph = Graph().parse(ont)
self.updates_ontology_reporter() # get finishing statistics
if self.bucket != '':
uploads_data_to_gcs_bucket(self.bucket, self.log_location,
log_dir, log)
log_str = '*** CLEANING MERGED ONTOLOGY DATA ***'
print('\n\n' + log_str)
logger.info(log_str)
self.ont_file_location = self.merged_ontology_filename
individual_ontologies = self.checks_for_downloaded_ontology_data()
self.merge_ontologies(individual_ontologies, self.temp_dir + '/',
self.ont_file_location)
if self.bucket != '':
uploads_data_to_gcs_bucket(self.bucket, self.log_location, log_dir,
log)
log_str = 'Loading Merged Ontology'
print('\n' + log_str)
logger.info(log_str)
self.ont_graph = Graph().parse(self.temp_dir + '/' +
self.ont_file_location)
self.updates_ontology_reporter() # get starting statistics
self.fixes_identifier_errors()
self.normalizes_duplicate_classes()
self.normalizes_existing_classes()
self.fixes_punning_errors()
self.updates_ontology_reporter() # get finishing statistics
if self.bucket != '':
uploads_data_to_gcs_bucket(self.bucket, self.log_location, log_dir,
log)
# serializes final ontology graph and uploads graph data and ontology report to gcs
self.ont_graph.serialize(destination=self.temp_dir + '/' +
self.ont_file_location,
format='xml')
ontology_file_formatter(self.temp_dir, '/' + self.ont_file_location,
self.owltools_location)
uploads_data_to_gcs_bucket(self.bucket, self.processed_data,
self.temp_dir, self.ont_file_location)
if self.bucket != '':
uploads_data_to_gcs_bucket(self.bucket, self.log_location, log_dir,
log)
log_str = '*** GENERATING ONTOLOGY CLEANING REPORT ***'
print('\n\n' + log_str)
logger.info(log_str)
self.generates_ontology_report()
if self.bucket != '':
uploads_data_to_gcs_bucket(self.bucket, self.log_location, log_dir,
log)
return None
def generates_ontology_report(self) -> None:
"""Parses the ontology_info dictionary in order to create a final ontology report summarizing the cleaning
results performed on each ontology and the statistics on the final merged set of ontologies.
Returns:
None.
"""
ontology_report_filename = 'ontology_cleaning_report.txt'
ont_order = sorted([
x for x in self.ontology_info.keys() if not x.startswith('Phe')
]) + [self.ont_file_location]
with open(self.temp_dir + '/' + ontology_report_filename, 'w') as o:
o.write('=' * 50 + '\n{}'.format('ONTOLOGY CLEANING REPORT'))
o.write('\n{}\n'.format(
str(datetime.datetime.utcnow().strftime('%a %b %d %X UTC %Y')))
+ '=' * 50 + '\n\n')
for key in ont_order:
o.write('\nONTOLOGY: {}\n'.format(key))
x = self.ontology_info[key]
if 'Original GCS URL' in x.keys():
o.write('\t- Original GCS URL: {}\n'.format(
x['Original GCS URL']))
if 'Processed GCS URL' in x:
o.write('\t- Processed GCS URL: {}\n'.format(
x['Processed GCS URL']))
o.write('\t- Statistics:\n\t\t- Before Cleaning: {}\n'.format(
x['Starting Statistics']))
if 'Final Statistics' in x.keys():
o.write('\t\t- After Cleaning: {}\n'.format(
x['Final Statistics']))
if 'ValueErrors' in x.keys():
o.write('\t- Value Errors: {}\n'.format(x['ValueErrors']))
if 'IdentifierErrors' in x.keys():
o.write('\t- Identifier Errors: {}\n'.format(
x['IdentifierErrors']))
if 'PheKnowLator_MergedOntologies' not in key:
if x['Deprecated'] != 'None':
o.write('\t- Deprecated Classes:\n')
for i in x['Deprecated']:
o.write('\t\t- {}\n'.format(str(i)))
else:
o.write('\t\t\t- {}\n'.format(x['Deprecated']))
if x['Obsolete'] != 'None':
o.write('\t- Obsolete Classes:\n')
for i in x['Obsolete']:
o.write('\t\t- {}\n'.format(str(i)))
else:
o.write('\t\t\t- {}\n'.format(x['Obsolete']))
o.write('\t- Punning Error:\n\t\t- Classes:\n')
if x['PunningErrors - Classes'] != 'None':
for i in x['PunningErrors - Classes'].split(', '):
o.write('\t\t\t- {}\n'.format(i))
else:
o.write('\t\t\t- {}\n'.format(
x['PunningErrors - Classes']))
o.write('\t\t- ObjectProperties:\n')
if x['PunningErrors - ObjectProperty'] != 'None':
for i in x['PunningErrors - ObjectProperty'].split(', '):
o.write('\t\t\t- {}\n'.format(i))
else:
o.write('\t\t\t- {}\n'.format(
x['PunningErrors - ObjectProperty']))
if 'Normalized - Duplicates' in x.keys():
o.write('\t- Entity Normalization:\n')
if x['Normalized - Duplicates'] != 'None':
for i in x['Normalized - Duplicates'].split(', '):
o.write('\t\t- {}\n'.format(i))
else:
o.write('\t\t- {}\n'.format(
x['Normalized - Duplicates']))
o.write(
'\t\t- Other Classes that May Need Normalization: {}\n'
.format(x['Normalized - NonOnt']))
o.write('\t\t- Normalized HGNC IDs: {}\n'.format(
x['Normalized - Gene IDs']))
o.write(
'\t- Deprecated Ontology HGNC Identifiers Needing Alignment:\n'
)
if x['Normalized - Dep'] != 'None':
for i in x['Normalized - Dep']:
o.write('\t\t- {}\n'.format(i))
else:
o.write('\t\t- {}\n'.format(x['Normalized - Dep']))
uploads_data_to_gcs_bucket(self.bucket, self.processed_data,
self.temp_dir, ontology_report_filename)
return None