def test_score_self(self):
# read in records
records = parse_input(
os.path.join(self.parent_dir, 'data/test.score-long.txt'),
self.hpo_network, self.alt2prim)
# limit to records with HPO terms since many test cases don't have the sub-graph terms from tests/data/hp.obo
input_records = [
x for x in records if x['record_id'] in ['213200', '302801']
]
results = self.scorer.score_records(
input_records, input_records,
half_product(len(input_records), len(input_records)))
self.assertEqual(len(results), 3)
# test the score of '213200' - '302801'
self.assertAlmostEqual(float(results[1][2]), 0.3758, 2)
def score(input_file,
output_file='-',
records_file=None,
annotations_file=None,
custom_disease_file=None,
ages_distribution_file=None,
self=False,
summarization_method='BMWA',
scoring_method='HRSS',
threads=1):
"""
Scores similarity of provided HPO annotated entries (see format below) against a set of HPO annotated dataset. By
default scoring happens against diseases annotated by the HPO group. See https://hpo.jax.org/app/download/annotation.
Phenopy also supports scoring the product of provided entries (see "--product") or scoring against a custom records
dataset (see "--records-file).
:param input_file: File with HPO annotated entries, one per line (see format below).
:param output_file: File path where to store the results. [default: - (stdout)]
:param records_file: An entity-to-phenotype annotation file in the same format as "input_file". This file, if
provided, is used to score entries in the "input_file" against entries here. [default: None]
:param annotations_file: An entity-to-phenotype annotation file in the same format as "input_file". This file, if
provided, is used to add information content to the network. [default: None]
:param custom_disease_file: entity Annotation for ranking diseases/genes
:param ages_distribution_file: Phenotypes age summary stats file containing phenotype HPO id, mean_age, and std.
[default: None]
:param self: Score entries in the "input_file" against itself.
:param summarization_method: The method used to summarize the HRSS matrix. Supported Values are best match average
(BMA), best match weighted average (BMWA), and maximum (maximum). [default: BMWA]
:param scoring_method: Either HRSS or Resnik
:param threads: Number of parallel processes to use. [default: 1]
"""
try:
obo_file = config.get('hpo', 'obo_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO OBO file found in the configuration file. See "hpo:obo_file" parameter.'
)
sys.exit(1)
if custom_disease_file is None:
try:
disease_to_phenotype_file = config.get(
'hpo', 'disease_to_phenotype_file')
except (NoSectionError, NoOptionError):
logger.critical(
'No HPO annotated dataset file found in the configuration file.'
' See "hpo:disease_to_phenotype_file" parameter.')
sys.exit(1)
else:
logger.info(
f"using custom disease annotation file: {custom_disease_file}")
disease_to_phenotype_file = custom_disease_file
logger.info(f'Loading HPO OBO file: {obo_file}')
hpo_network, alt2prim, disease_records = \
generate_annotated_hpo_network(obo_file,
disease_to_phenotype_file,
annotations_file=annotations_file,
ages_distribution_file=ages_distribution_file
)
# parse input records
input_records = parse_input(input_file, hpo_network, alt2prim)
# create instance the scorer class
try:
scorer = Scorer(hpo_network,
summarization_method=summarization_method,
scoring_method=scoring_method)
except ValueError as e:
logger.critical(f'Failed to initialize scoring class: {e}')
sys.exit(1)
if self:
score_records = input_records
scoring_pairs = half_product(len(score_records), len(score_records))
else:
if records_file:
score_records = parse_input(records_file, hpo_network, alt2prim)
else:
score_records = disease_records
scoring_pairs = itertools.product(
range(len(input_records)),
range(len(score_records)),
)
results = scorer.score_records(input_records, score_records, scoring_pairs,
threads)
with open_or_stdout(output_file) as output_fh:
output_fh.write('\t'.join(['#query', 'entity_id', 'score']))
output_fh.write('\n')
for result in results:
output_fh.write('\t'.join(str(column) for column in result))
output_fh.write('\n')
def run_phenoseries_experiment(outdir=None,
phenotypic_series_filepath=None,
min_hpos=2,
min_entities=4,
phenoseries_fraction=1.0,
scoring_method="HRSS",
threads=1,
omim_phenotypes_file=None,
pairwise_mim_scores_file=None):
if outdir is None:
outdir = os.getcwd
# load HPO network
# data directory
phenopy_data_directory = os.path.join(os.getenv("HOME"), ".phenopy/data")
# files used in building the annotated HPO network
obo_file = os.path.join(phenopy_data_directory, "hp.obo")
disease_to_phenotype_file = os.path.join(phenopy_data_directory,
"phenotype.hpoa")
hpo_network, alt2prim, _ = generate_annotated_hpo_network(
obo_file, disease_to_phenotype_file, ages_distribution_file=None)
# read the phenotypic series file as a DataFrame
psdf = pd.read_csv(
phenotypic_series_filepath,
sep="\t",
comment="#",
names=["PS", "MIM", "Phenotype"],
)
# null phenotypes are actually null MIM id fields, so just drop these
psdf = psdf.dropna().sample(frac=phenoseries_fraction, random_state=42)
psdf.reset_index(inplace=True, drop=True)
# create a dictionary for phenotypic series to list of omim ids mapping
ps2mimids = {}
for ps, mim_ids in psdf.groupby(["PS"])["MIM"]:
# more than two mims in a ps
if len(mim_ids) >= 2:
ps2mimids[ps] = list(set([int(mid) for mid in mim_ids.tolist()]))
# invert the ps2mimid dictionary for easy lookup of which ps a mim belongs to
mim2psids = {}
for mim_id, ps in psdf.groupby(["MIM"])["PS"]:
mim2psids[int(mim_id)] = ps.tolist()
fields_to_use = [
"text",
"description",
"otherFeatures",
"biochemicalFeatures",
"diagnosis",
"clinicalFeatures",
]
if omim_phenotypes_file == "":
logger.info("Scraping OMIM Diseases text")
mim_texts = {}
for mim_id in mim2psids:
mim_response = request_mimid_info(mim_id)
try:
mim_info = mim_response.json()
except AttributeError:
break
mim_text = mim_info["omim"]["entryList"][0]["entry"][
"textSectionList"]
all_mim_text = ""
for text_section in mim_text:
section_name = text_section["textSection"]["textSectionName"]
if section_name in fields_to_use:
# unique_section_names.add(section_name)
all_mim_text += " " + text_section["textSection"][
"textSectionContent"]
mim_texts[mim_id] = all_mim_text
# instantiate txt2hpo's Exctractor class to perform named entity recognition
extractor = Extractor(remove_negated=True,
max_neighbors=3,
correct_spelling=False)
# loop over the MIM ids and extract hpo ids from each MIM's text fields
mim_hpos = {}
for mim_id in mim2psids:
mim_hpos[mim_id] = extractor.hpo(mim_texts[mim_id]).hpids
mimdf = pd.DataFrame()
mimdf["omim_id"] = list(mim2psids.keys())
mimdf["hpo_terms"] = mimdf["omim_id"].apply(
lambda mim_id: mim_hpos[mim_id])
mimdf.to_csv(os.path.join(outdir, "omim_phenotypes.txt"),
index=False,
sep='\t')
else:
logger.info("You passed an OMIM disease to phenotype file")
try:
mimdf = pd.read_csv(omim_phenotypes_file, sep="\t")
mimdf["omim_id"] = mimdf["omim_id"].astype(int)
mimdf["hpo_terms"] = mimdf["hpo_terms"].apply(literal_eval)
mim_hpos = dict(zip(mimdf["omim_id"], mimdf["hpo_terms"]))
except FileNotFoundError:
sys.exit("Please provide a valid file path")
# do we need this?
# mim_hpos = {mim_id: hpos for mim_id, hpos in mim_hpos.items()}
# clean up HPO ids in lists
for mim_id, hpo_ids in mim_hpos.items():
mim_hpos[mim_id] = convert_and_filter_hpoids(hpo_ids, hpo_network,
alt2prim)
# remove entities (mims) that have less than min_hpos
mims_to_remove = []
for mim_id, hpo_ids in mim_hpos.copy().items():
if len(hpo_ids) <= min_hpos:
mims_to_remove.append(mim_id)
# Now remove the entities (mim ids) with less than min_hpos
experiment_ps2mimids = {}
# remove these mims from ps
for ps, mimids in ps2mimids.copy().items():
experiment_ps2mimids[ps] = []
for ps_mim_id in mimids:
if ps_mim_id not in mims_to_remove:
experiment_ps2mimids[ps].append(ps_mim_id)
# After removing entities, make sure the series has min number of entities
# get lists of mims and their PS
remove_these_ps = []
for ps, mimids in experiment_ps2mimids.items():
if len(mimids) < min_entities:
remove_these_ps.append(ps)
for psid in remove_these_ps:
del experiment_ps2mimids[psid]
# Create a unique list of entity ids, for scoring later
experiment_omims = set()
for psid, mim_ids in experiment_ps2mimids.items():
for mim in mim_ids:
experiment_omims.add(mim)
experiment_omims = list(experiment_omims)
# make a DataFrame for entity ids
mimdf = pd.DataFrame()
mimdf["omim_id"] = experiment_omims
mimdf["hpo_terms"] = mimdf["omim_id"].apply(
lambda mim_id: mim_hpos[mim_id])
if pairwise_mim_scores_file == "":
scorer = Scorer(hpo_network, scoring_method=scoring_method)
records = [{
"record_id":
mim_id,
"terms":
convert_and_filter_hpoids(hpo_terms, hpo_network, alt2prim),
"weights": {},
} for mim_id, hpo_terms in dict(
zip(mimdf["omim_id"], mimdf["hpo_terms"])).items()]
results = scorer.score_records(records,
records,
half_product(len(records),
len(records)),
threads=threads)
pairwise_scores = pd.DataFrame(
results, columns=["mimid1", "mimid2", "phenopy-score"])
# convert to square form
pairwise_scores = pairwise_scores.set_index(["mimid1",
"mimid2"]).unstack()
# This pandas method chain fills in the missing scores of the square matrix with the values from the transpose of df.
pairwise_scores = (pairwise_scores["phenopy-score"].reset_index(
drop=True).fillna(
pairwise_scores.T.droplevel(0).reset_index(
drop=True)).set_index(pairwise_scores.index, drop=True))
# reindex with the mimdf index
pairwise_scores = pairwise_scores.reindex(mimdf["omim_id"].tolist())
pairwise_scores = pairwise_scores[mimdf["omim_id"].tolist()]
pd.DataFrame(pairwise_scores).to_csv(os.path.join(
outdir, 'phenoseries.psim_matrix.txt'),
sep='\t')
else:
pairwise_scores = pd.read_csv(pairwise_mim_scores_file, sep='\t')
ranksdf = make_rank_dataframe(
pairwise_scores.astype(float).values, mimdf, experiment_ps2mimids)
ranksdf.to_csv(os.path.join(outdir, "phenoseries.rankdf.txt"), sep="\t")