def predict_likelihood_moldx(phenotypes,
phenotype_groups=None,
hpo_network=None,
alt2prim=None,
k_phenotype_groups=1000):
"""
Predicts the likelihood of molecular diagnosis given a set of phenotypes.
:param phenotypes: A list of phenotypes or a list of lists of phenotypes.
:param phenotype_groups: <optionnal> A dictionary of phenotype to phenotype group mappings.
:param hpo_network: <optional> The hpo networkx object.
:param alt2prim: <optional> A dictionary of alternate phenotype ids to primary phenotype ids. (must be given if hpo_network is provided)
:param k_phenotype_groups <optional> An integer that represents the number of phenotype groups to use.
:return: An array of probabilities for the positive class.
"""
# detect if phenotypes is 1d or 2d
if hpo_network is None or alt2prim is None:
try:
obo_file = config.get('hpo', 'obo_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO OBO file found in the configuration file. See "hpo:obo_file" parameter.'
)
raise
try:
disease_to_phenotype_file = config.get(
'hpo', 'disease_to_phenotype_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO annotated dataset file found in the configuration file.'
' See "hpo:disease_to_phenotype_file" parameter.')
raise
logger.info(f'Loading HPO OBO file: {obo_file}')
hpo_network, alt2prim, _ = \
generate_annotated_hpo_network(obo_file,
disease_to_phenotype_file,
)
if phenotype_groups is None:
phenotype_groups = read_phenotype_groups()
try:
phenotype_groups[list(phenotype_groups)[0]][f"k{k_phenotype_groups}"]
except KeyError:
logger.critical(
"The value for k_phenotype_groups was not valid. Please use a valid k from the phenotype_groups dictionary."
)
raise
encoded_phenotypes = encode_phenotypes(phenotypes,
phenotype_groups,
hpo_network,
alt2prim,
k=k_phenotype_groups)
model = joblib.load(config['models']['likelihood.model'])
probabilities = model.predict_proba(encoded_phenotypes)
return probabilities[:, 1]
def likelihood_moldx(input_file, output_file=None, k_phenotype_groups=1000):
"""
:param input_file: The file path to a file containing three columns. [ID\tkey=value\thpodid,hpoid,hpoid]
:param output_file: The file path to an output file containing the predicted probabilities
:param k_phenotype_groups: The number of phenotype groups to use for encoding phenotypes. The CLI version of phenopy allows for one of [1000, 1500]
"""
try:
obo_file = config.get('hpo', 'obo_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO OBO file found in the configuration file. See "hpo:obo_file" parameter.'
)
sys.exit(1)
try:
disease_to_phenotype_file = config.get('hpo',
'disease_to_phenotype_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO annotated dataset file found in the configuration file.'
' See "hpo:disease_to_phenotype_file" parameter.')
sys.exit(1)
logger.info(f'Loading HPO OBO file: {obo_file}')
hpo_network, alt2prim, _ = \
generate_annotated_hpo_network(obo_file,
disease_to_phenotype_file,
)
# parse input records
input_records = parse_input(input_file, hpo_network, alt2prim)
record_ids = [record["record_id"] for record in input_records]
phenotypes = [record["terms"] for record in input_records]
# predict likelihood of molecular diagnosis
positive_probabilities = predict_likelihood_moldx(
phenotypes,
phenotype_groups=None,
hpo_network=hpo_network,
alt2prim=alt2prim,
k_phenotype_groups=k_phenotype_groups,
)
if output_file is None:
output_file = "phenopy.likelihood_moldx.txt"
try:
with open(output_file, "w") as f:
for sample_id, probability in zip(record_ids,
positive_probabilities):
f.write(f"{sample_id}\t{probability}\n")
except IOError:
sys.exit("Something went wrong writing the probabilities to file")
def setUp(cls):
# parent dir
cls.parent_dir = os.path.dirname(os.path.realpath(__file__))
if 'hpo' not in config.sections():
config.add_section('hpo')
config.set('hpo', 'obo_file', os.path.join(cls.parent_dir, 'data/hp.obo'))
config.set('hpo', 'disease_to_phenotype_file', os.path.join(cls.parent_dir, 'data/phenotype.hpoa'))
cls.obo_file = config.get('hpo', 'obo_file')
cls.disease_to_phenotype_file = config.get('hpo', 'disease_to_phenotype_file')
cls.hpo_network, cls.alt2prim, cls.disease_records = generate_annotated_hpo_network(
cls.obo_file,
cls.disease_to_phenotype_file,
)
cls.phenotype_groups = read_phenotype_groups()
def __init__(self, hpo_network, summarization_method='BMWA', min_score_mask=0.05,
scoring_method='HRSS'):
self.hpo_network = hpo_network
if summarization_method not in ['BMA', 'BMWA', 'maximum']:
raise ValueError('Unsupported summarization method, please choose from BMA, BMWA, or maximum.')
self.summarization_method = summarization_method
self.min_score_mask = min_score_mask
if scoring_method not in ['HRSS', 'Resnik', 'Jaccard', 'word2vec']:
raise ValueError('Unsupported semantic similarity scoring method, please choose from HRSS, Resnik, Jaccard, or word2vec.')
self.scoring_method = scoring_method
if scoring_method == 'word2vec':
try:
self.word_vectors = gensim.models.KeyedVectors.load(config.get('models', 'phenopy.wv.model'))
except FileNotFoundError:
raise ValueError("Please make sure that a word2vec model is in your project data directory.")
def request_mimid_info(mimid):
"""
request mimid description from OMIM
"""
access = "entry?"
api_key = os.getenv("OMIM_API_KEY")
if api_key is None:
api_key = config.get("omim", "omim_api_key")
payload = {
"mimNumber": mimid,
"include": "text",
"format": "json",
"apiKey": api_key,
}
r = requests.get(OMIM_API_URL + access, params=payload)
if r.status_code == 200:
return r
else:
logger.critical(
"Please set the omim_api_key in your phenopy.ini config file")
def _load_hpo_network(obo_file,
terms_to_genes,
annotations_count,
custom_annotations_file,
hpo_network_file=None):
"""
Load and process phenotypes to genes and obo files if we don't have a processed network already.
"""
# We instruct the user that they can set hpo_network_file in .phenopy/phenopy.ini
# The default value is empty string, so check for that first.
if hpo_network_file is None:
hpo_network_file = config.get('hpo', 'hpo_network_file')
if not os.path.exists(hpo_network_file):
# load and process hpo network
logger.info(f'Loading HPO OBO file: {obo_file}')
hpo_network = load_obo(obo_file, logger=logger)
hpo_network = process(hpo_network,
terms_to_genes,
annotations_count,
custom_annotations_file,
logger=logger)
# save a cache of the processed network
cache(hpo_network, hpo_network_file)
# the default hpo_network.pickle file was found
else:
try:
hpo_network = restore(hpo_network_file)
except (FileNotFoundError, PermissionError, IsADirectoryError) as e:
logger.critical(
f'{hpo_network_file} is not a valid path to a pickled hpo_network file.\n'
f'In your $HOME/.phenopy/phenopy.ini, please set hpo_network_file'
f'=/path/to/hpo_netowrk.pickle OR leave it empty, which is the default. '
)
raise e
return hpo_network
def score(input_file,
output_file='-',
records_file=None,
annotations_file=None,
custom_disease_file=None,
ages_distribution_file=None,
self=False,
summarization_method='BMWA',
scoring_method='HRSS',
threads=1):
"""
Scores similarity of provided HPO annotated entries (see format below) against a set of HPO annotated dataset. By
default scoring happens against diseases annotated by the HPO group. See https://hpo.jax.org/app/download/annotation.
Phenopy also supports scoring the product of provided entries (see "--product") or scoring against a custom records
dataset (see "--records-file).
:param input_file: File with HPO annotated entries, one per line (see format below).
:param output_file: File path where to store the results. [default: - (stdout)]
:param records_file: An entity-to-phenotype annotation file in the same format as "input_file". This file, if
provided, is used to score entries in the "input_file" against entries here. [default: None]
:param annotations_file: An entity-to-phenotype annotation file in the same format as "input_file". This file, if
provided, is used to add information content to the network. [default: None]
:param custom_disease_file: entity Annotation for ranking diseases/genes
:param ages_distribution_file: Phenotypes age summary stats file containing phenotype HPO id, mean_age, and std.
[default: None]
:param self: Score entries in the "input_file" against itself.
:param summarization_method: The method used to summarize the HRSS matrix. Supported Values are best match average
(BMA), best match weighted average (BMWA), and maximum (maximum). [default: BMWA]
:param scoring_method: Either HRSS or Resnik
:param threads: Number of parallel processes to use. [default: 1]
"""
try:
obo_file = config.get('hpo', 'obo_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO OBO file found in the configuration file. See "hpo:obo_file" parameter.'
)
sys.exit(1)
if custom_disease_file is None:
try:
disease_to_phenotype_file = config.get(
'hpo', 'disease_to_phenotype_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO annotated dataset file found in the configuration file.'
' See "hpo:disease_to_phenotype_file" parameter.')
sys.exit(1)
else:
logger.info(
f"using custom disease annotation file: {custom_disease_file}")
disease_to_phenotype_file = custom_disease_file
logger.info(f'Loading HPO OBO file: {obo_file}')
hpo_network, alt2prim, disease_records = \
generate_annotated_hpo_network(obo_file,
disease_to_phenotype_file,
annotations_file=annotations_file,
ages_distribution_file=ages_distribution_file
)
# parse input records
input_records = parse_input(input_file, hpo_network, alt2prim)
# create instance the scorer class
try:
scorer = Scorer(hpo_network,
summarization_method=summarization_method,
scoring_method=scoring_method)
except ValueError as e:
logger.critical(f'Failed to initialize scoring class: {e}')
sys.exit(1)
if self:
score_records = input_records
scoring_pairs = half_product(len(score_records), len(score_records))
else:
if records_file:
score_records = parse_input(records_file, hpo_network, alt2prim)
else:
score_records = disease_records
scoring_pairs = itertools.product(
range(len(input_records)),
range(len(score_records)),
)
results = scorer.score_records(input_records, score_records, scoring_pairs,
threads)
with open_or_stdout(output_file) as output_fh:
output_fh.write('\t'.join(['#query', 'entity_id', 'score']))
output_fh.write('\n')
for result in results:
output_fh.write('\t'.join(str(column) for column in result))
output_fh.write('\n')
def score(query_hpo_file, records_file=None, query_name='SAMPLE', obo_file=None, pheno2genes_file=None, threads=1,
agg_score='BMA', no_parents=False, custom_annotations_file=None, output_file=None):
"""
Scores a case HPO terms against all genes associated HPO.
:param query_hpo_file: File with case HPO terms, one per line.
:param records_file: One record per line, tab delimited. First column record unique identifier, second column
pipe separated list of HPO identifier (HP:0000001).
:param query_name: Unique identifier for the query file.
:param obo_file: OBO file from https://hpo.jax.org/app/download/ontology.
:param pheno2genes_file: Phenotypes to genes from https://hpo.jax.org/app/download/annotation.
:param threads: Number of parallel process to use.
:param agg_score: The aggregation method to use for summarizing the similarity matrix between two term sets
Must be one of {'BMA', 'maximum'}
:param no_parents: If provided, scoring is done by only using the most informative nodes. All parent nodes are removed.
:param custom_annotations_file: A custom entity-to-phenotype annotation file in the same format as tests/data/test.score-product.txt
:param output_file: filepath where to store the results.
"""
if agg_score not in {'BMA', 'maximum', }:
logger.critical(
'agg_score must be one of {BMA, maximum}.')
exit(1)
if obo_file is None:
try:
obo_file = config.get('hpo', 'obo_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO OBO file provided and no "hpo:obo_file" found in the configuration file.')
exit(1)
if pheno2genes_file is None:
try:
pheno2genes_file = config.get('hpo', 'pheno2genes_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO pheno2genes_file file provided and no "hpo:pheno2genes_file" found in the configuration file.'
)
exit(1)
try:
with open(query_hpo_file, 'r') as case_fh:
case_hpo = case_fh.read().splitlines()
except (FileNotFoundError, PermissionError) as e:
logger.critical(e)
exit(1)
# load phenotypes to genes associations
terms_to_genes, genes_to_terms, annotations_count = load_p2g(
pheno2genes_file, logger=logger)
# load hpo network
hpo_network = _load_hpo_network(
obo_file, terms_to_genes, annotations_count, custom_annotations_file)
# create instance the scorer class
scorer = Scorer(hpo_network)
# multiprocessing objects
manager = Manager()
lock = manager.Lock()
if no_parents is True:
case_hpo = remove_parents(case_hpo, hpo_network)
if records_file:
# score and output case hpo terms against all genes associated set of hpo terms
logger.info(
f'Scoring HPO terms from file: {query_hpo_file} against entities in: {records_file}')
records = read_records_file(records_file, no_parents, hpo_network, logger=logger)
# include the case-to-iteslf
records[query_name] = case_hpo
if not output_file:
sys.stdout.write('\t'.join(['#query', 'entity_id', 'score']))
sys.stdout.write('\n')
with Pool(threads) as p:
p.starmap(scorer.score_pairs, [(records, [
(query_name, record) for record in records], lock, agg_score, i, threads) for i in range(threads)])
else:
with Pool(threads) as p:
scored_results = p.starmap(scorer.score_pairs, [(records, [(query_name, record) for record in records],
lock, agg_score, i, threads, False) for i in range(threads)])
scored_results = [item for sublist in scored_results for item in sublist]
scored_results_df = pd.DataFrame(data=scored_results, columns='#query,entity_id,score'.split(','))
scored_results_df = scored_results_df.sort_values(by='score', ascending=False)
scored_results_df.to_csv(output_file, sep='\t', index=False)
logger.info(f'Scoring completed')
logger.info(f'Writing results to file: {output_file}')
else:
# score and output case hpo terms against all genes associated set of hpo terms
logger.info(f'Scoring case HPO terms from file: {query_hpo_file}')
# add the case terms to the genes_to_terms dict
genes_to_terms[query_name] = case_hpo
if not output_file:
sys.stdout.write('\t'.join(['#query', 'gene', 'score']))
sys.stdout.write('\n')
# iterate over each cross-product and score the pair of records
with Pool(threads) as p:
p.starmap(scorer.score_pairs, [(genes_to_terms, [
(query_name, gene) for gene in genes_to_terms], lock, agg_score, i, threads) for i in range(threads)])
else:
with Pool(threads) as p:
scored_results = p.starmap(scorer.score_pairs, [(genes_to_terms,
[(query_name, gene) for gene in genes_to_terms], lock, agg_score, i, threads, False)
for i in range(threads)])
scored_results = [item for sublist in scored_results for item in sublist]
scored_results_df = pd.DataFrame(data=scored_results, columns='#query,gene,score'.split(','))
scored_results_df = scored_results_df.sort_values(by='score', ascending=False)
scored_results_df.to_csv(output_file, sep='\t', index=False)
logger.info(f'Scoring completed')
logger.info(f'Writing results to file: {output_file}')
def score_product(records_file, obo_file=None, pheno2genes_file=None, threads=1, agg_score='BMA', no_parents=False,
custom_annotations_file=None):
"""
Scores the cartesian product of HPO terms from a list of unique records (cases, genes, diseases, etc).
:param records_file: One record per line, tab delimited. First column record unique identifier, second column
pipe separated list of HPO identifier (HP:0000001).
:param obo_file: OBO file from https://hpo.jax.org/app/download/ontology.
:param pheno2genes_file: Phenotypes to genes from https://hpo.jax.org/app/download/annotation.
:param threads: Multiprocessing threads to use [default: 1].
:param agg_score: The aggregation method to use for summarizing the similarity matrix between two term sets
Must be one of {'BMA', 'maximum'}
:param no_parents: If provided, scoring is done by only using the most informative nodes. All parent nodes are removed.
:param custom_annotations_file: A custom entity-to-phenotype annotation file in the same format as tests/data/test.score-product.txt
"""
if agg_score not in {'BMA', 'maximum', }:
logger.critical(
'agg_score must be one of {BMA, maximum}.')
exit(1)
if obo_file is None:
try:
obo_file = config.get('hpo', 'obo_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO OBO file provided and no "hpo:obo_file" found in the configuration file.')
exit(1)
if pheno2genes_file is None:
try:
pheno2genes_file = config.get('hpo', 'pheno2genes_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO pheno2genes_file file provided and no "hpo:pheno2genes_file" found in the configuration file.'
)
exit(1)
# load phenotypes to genes associations
terms_to_genes, _, annotations_count = load_p2g(
pheno2genes_file, logger=logger)
# load hpo network
hpo_network = _load_hpo_network(
obo_file, terms_to_genes, annotations_count, custom_annotations_file)
# try except
records = read_records_file(records_file, no_parents, hpo_network, logger=logger)
logger.info(f'Scoring product of records from file: {records_file}')
# create instance the scorer class
scorer = Scorer(hpo_network)
# create records product generator
records_product = itertools.product(records.keys(), repeat=2)
# iterate over each cross-product and score the pair of records
manager = Manager()
lock = manager.Lock()
with Pool(threads) as p:
p.starmap(scorer.score_pairs, [(records, records_product,
lock, agg_score, i, threads) for i in range(threads)])