def test_explode_on_set(self):
t = hl.utils.range_table(1)
t = t.annotate(a=hl.set(['a', 'b', 'c']))
t = t.explode('a')
self.assertEqual(set(t.collect()),
hl.eval(hl.set([hl.struct(idx=0, a='a'),
hl.struct(idx=0, a='b'),
hl.struct(idx=0, a='c')])))
def test_explode_on_set(self):
t = hl.utils.range_table(1)
t = t.annotate(a=hl.set(['a', 'b', 'c']))
t = t.explode('a')
self.assertEqual(set(t.collect()),
hl.eval(hl.set([hl.struct(idx=0, a='a'),
hl.struct(idx=0, a='b'),
hl.struct(idx=0, a='c')])))
def test_group_by_field_lifetimes(self):
mt = hl.utils.range_matrix_table(3, 3)
mt2 = (mt.group_rows_by(row_idx='100').aggregate(
x=hl.agg.collect_as_set(mt.row_idx + 5)))
assert mt2.aggregate_entries(hl.agg.all(mt2.x == hl.set({5, 6, 7})))
mt3 = (mt.group_cols_by(col_idx='100').aggregate(
x=hl.agg.collect_as_set(mt.col_idx + 5)))
assert mt3.aggregate_entries(hl.agg.all(mt3.x == hl.set({5, 6, 7})))
def test_group_by_field_lifetimes(self):
mt = hl.utils.range_matrix_table(3, 3)
mt2 = (mt.group_rows_by(row_idx='100')
.aggregate(x=hl.agg.collect_as_set(mt.row_idx + 5)))
assert mt2.aggregate_entries(hl.agg.all(mt2.x == hl.set({5, 6, 7})))
mt3 = (mt.group_cols_by(col_idx='100')
.aggregate(x=hl.agg.collect_as_set(mt.col_idx + 5)))
assert mt3.aggregate_entries(hl.agg.all(mt3.x == hl.set({5, 6, 7})))
def test_multiple_files_variant_filtering(self):
bgen_file = [resource('random-b.bgen'), resource('random-c.bgen'), resource('random-a.bgen')]
hl.index_bgen(bgen_file)
alleles = ['A', 'G']
desired_variants = [
hl.Struct(locus=hl.Locus('20', 11), alleles=alleles),
hl.Struct(locus=hl.Locus('20', 13), alleles=alleles),
hl.Struct(locus=hl.Locus('20', 29), alleles=alleles),
hl.Struct(locus=hl.Locus('20', 28), alleles=alleles),
hl.Struct(locus=hl.Locus('20', 1), alleles=alleles),
hl.Struct(locus=hl.Locus('20', 12), alleles=alleles),
]
actual = hl.import_bgen(bgen_file,
['GT'],
n_partitions=10,
variants=desired_variants)
self.assertEqual(actual.count_rows(), 6)
everything = hl.import_bgen(bgen_file,
['GT'])
self.assertEqual(everything.count(), (30, 10))
expected = everything.filter_rows(hl.set(desired_variants).contains(everything.row_key))
self.assertTrue(expected._same(actual))
def pop_max_expr(
freq: hl.expr.ArrayExpression,
freq_meta: hl.expr.ArrayExpression,
pops_to_exclude: Optional[Set[str]] = None,
) -> hl.expr.StructExpression:
"""
Creates an expression containing popmax: the frequency information about the population
that has the highest AF from the populations provided in `freq_meta`,
excluding those specified in `pops_to_exclude`.
Only frequencies from adj populations are considered.
This resulting struct contains the following fields:
- AC: int32
- AF: float64
- AN: int32
- homozygote_count: int32
- pop: str
:param freq: ArrayExpression of Structs with fields ['AC', 'AF', 'AN', 'homozygote_count']
:param freq_meta: ArrayExpression of meta dictionaries corresponding to freq (as returned by annotate_freq)
:param pops_to_exclude: Set of populations to skip for popmax calcluation
:return: Popmax struct
"""
_pops_to_exclude = hl.literal(pops_to_exclude)
popmax_freq_indices = hl.range(0, hl.len(freq_meta)).filter(
lambda i: (hl.set(freq_meta[i].keys()) == {"group", "pop"})
& (freq_meta[i]["group"] == "adj")
& (~_pops_to_exclude.contains(freq_meta[i]["pop"])))
freq_filtered = popmax_freq_indices.map(lambda i: freq[i].annotate(
pop=freq_meta[i]["pop"])).filter(lambda f: f.AC > 0)
sorted_freqs = hl.sorted(freq_filtered, key=lambda x: x.AF, reverse=True)
return hl.or_missing(hl.len(sorted_freqs) > 0, sorted_freqs[0])
def merge_alleles(alleles):
from hail.expr.functions import _num_allele_type, _allele_ints
return hl.rbind(
alleles.map(lambda a: hl.or_else(a[0], ''))
.fold(lambda s, t: hl.cond(hl.len(s) > hl.len(t), s, t), ''),
lambda ref:
hl.rbind(
alleles.map(
lambda al: hl.rbind(
al[0],
lambda r:
hl.array([ref]).extend(
al[1:].map(
lambda a:
hl.rbind(
_num_allele_type(r, a),
lambda at:
hl.cond(
(_allele_ints['SNP'] == at) |
(_allele_ints['Insertion'] == at) |
(_allele_ints['Deletion'] == at) |
(_allele_ints['MNP'] == at) |
(_allele_ints['Complex'] == at),
a + ref[hl.len(r):],
a)))))),
lambda lal:
hl.struct(
globl=hl.array([ref]).extend(hl.array(hl.set(hl.flatten(lal)).remove(ref))),
local=lal)))
def filter_to_clinvar_pathogenic(
t: Union[hl.MatrixTable, hl.Table],
clnrevstat_field: str = "CLNREVSTAT",
clnsig_field: str = "CLNSIG",
clnsigconf_field: str = "CLNSIGCONF",
remove_no_assertion: bool = True,
remove_conflicting: bool = True,
) -> Union[hl.MatrixTable, hl.Table]:
"""
Return a MatrixTable or Table that filters the clinvar data to pathogenic and likely pathogenic variants.
Example use:
.. code-block:: python
from gnomad.resources.grch38.reference_data import clinvar
clinvar_ht = clinvar.ht()
clinvar_ht = filter_to_clinvar_pathogenic(clinvar_ht)
:param: t: Input dataset that contains clinvar data, could either be a MatrixTable or Table.
:param clnrevstat_field: The field string for the expression that contains the review status of the clinical significance of clinvar variants.
:param clnsig_field: The field string for the expression that contains the clinical signifcance of the clinvar variant.
:param clnsigconf_field: The field string for the expression that contains the conflicting clinical significance values for the variant. For variants with no conflicting significance, this field should be undefined.
:param remove_no_assertion: Flag for removing entries in which the clnrevstat (clinical significance) has no assertions (zero stars).
:param remove_conflicting: Flag for removing entries with conflicting clinical interpretations.
:return: Filtered MatrixTable or Table
"""
logger.info(
"Found %d variants before filtering",
t.count_rows() if isinstance(t, hl.MatrixTable) else t.count(),
)
path_expr = (t.info[clnsig_field].map(lambda x: x.lower()).map(
lambda x: x.contains("pathogenic")).any(lambda x: x))
if remove_no_assertion:
logger.info("Variants without assertions will be removed.")
no_star_assertions = hl.literal({
"no_assertion_provided",
"no_assertion_criteria_provided",
"no_interpretation_for_the_individual_variant",
})
path_expr = path_expr & (hl.set(t.info[clnrevstat_field]).intersection(
no_star_assertions).length() == 0)
if remove_conflicting:
logger.info(
"Variants with conflicting clinical interpretations will be removed."
)
path_expr = path_expr & hl.is_missing(t.info[clnsigconf_field])
if isinstance(t, hl.MatrixTable):
t = t.filter_rows(path_expr)
else:
t = t.filter(path_expr)
logger.info(
"Found %d variants after filtering to clinvar pathogenic variants.",
t.count_rows() if isinstance(t, hl.MatrixTable) else t.count(),
)
return t
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--base-level-pext",
help="Path to Hail table with base-level data",
default=
"gs://gnomad-public/papers/2019-tx-annotation/gnomad_browser/all.baselevel.021620.ht",
)
parser.add_argument(
"--low-max-pext-genes",
help="Path to table containing list of genes with low max pext",
default=
"gs://gnomad-public/papers/2019-tx-annotation/data/GRCH37_hg19/max_pext_low_genes.021520.tsv",
)
parser.add_argument(
"output_path", help="Path to output Hail table with region-level data")
args = parser.parse_args()
ds = prepare_pext_data(args.base_level_pext)
low_max_pext_genes = hl.import_table(args.low_max_pext_genes)
low_max_pext_genes = low_max_pext_genes.aggregate(
hl.agg.collect_as_set(low_max_pext_genes.ensg))
ds = ds.annotate(flags=hl.cond(
hl.set(low_max_pext_genes).contains(ds.gene_id),
hl.literal(["low_max_pext"]),
hl.empty_array(hl.tstr),
))
ds.write(args.output_path)
def get_mt_filtered_by_pops(pops: list,
chrom: str = 'all',
imputed: bool = True,
min_mac: int = 20,
entry_fields=('GP', ),
filter_mac_instead_of_ac: bool = False):
r'''
Wraps `get_filtered_mt()` from ukbb_pan_ancestry.resources.genotypes
This filters to samples from populations listed in `pops`.
NOTE: If chrom='all', this loads all autosomes AND chrX.
'''
assert len(pops) > 0 and set(pops).issubset(POPS)
kwargs = {
'pop': 'all' if len(pops) > 1 else pops[0],
'imputed': imputed,
'min_mac': min_mac,
'entry_fields': entry_fields,
'filter_mac_instead_of_ac': filter_mac_instead_of_ac
}
mt = get_filtered_mt(chrom=chrom,
**kwargs) # in this case chrom='all' gets autosomes
if chrom == 'all':
mt_x = get_filtered_mt(chrom='X', **kwargs)
mt = mt.union_rows(mt_x)
if len(pops) > 1:
mt = mt.filter_cols(hl.set(pops).contains(mt.pop))
return mt
def merge_alleles(alleles):
from hail.expr.functions import _num_allele_type, _allele_ints
return hl.rbind(
alleles.map(lambda a: hl.or_else(a[0], ''))
.fold(lambda s, t: hl.cond(hl.len(s) > hl.len(t), s, t), ''),
lambda ref:
hl.rbind(
alleles.map(
lambda al: hl.rbind(
al[0],
lambda r:
hl.array([ref]).extend(
al[1:].map(
lambda a:
hl.rbind(
_num_allele_type(r, a),
lambda at:
hl.cond(
(_allele_ints['SNP'] == at) |
(_allele_ints['Insertion'] == at) |
(_allele_ints['Deletion'] == at) |
(_allele_ints['MNP'] == at) |
(_allele_ints['Complex'] == at),
a + ref[hl.len(r):],
a)))))),
lambda lal:
hl.struct(
globl=hl.array([ref]).extend(hl.array(hl.set(hl.flatten(lal)).remove(ref))),
local=lal)))
def test_multiple_files_variant_filtering(self):
bgen_file = [
resource('random-b.bgen'),
resource('random-c.bgen'),
resource('random-a.bgen')
]
hl.index_bgen(bgen_file)
alleles = ['A', 'G']
desired_variants = [
hl.Struct(locus=hl.Locus('20', 11), alleles=alleles),
hl.Struct(locus=hl.Locus('20', 13), alleles=alleles),
hl.Struct(locus=hl.Locus('20', 29), alleles=alleles),
hl.Struct(locus=hl.Locus('20', 28), alleles=alleles),
hl.Struct(locus=hl.Locus('20', 1), alleles=alleles),
hl.Struct(locus=hl.Locus('20', 12), alleles=alleles),
]
actual = hl.import_bgen(bgen_file, ['GT'],
n_partitions=10,
variants=desired_variants)
self.assertEqual(actual.count_rows(), 6)
everything = hl.import_bgen(bgen_file, ['GT'])
self.assertEqual(everything.count(), (30, 10))
expected = everything.filter_rows(
hl.set(desired_variants).contains(everything.row_key))
self.assertTrue(expected._same(actual))
def collect_gene_exons(gene_exons):
# There are 3 feature types in the exons collection: "CDS", "UTR", and "exon".
# There are "exon" regions that cover the "CDS" and "UTR" regions and also
# some (non-coding) transcripts that contain only "exon" regions.
# This filters the "exon" regions to only those that are in non-coding transcripts.
#
# This makes the UI for selecting visible regions easier, since it can filter
# on "CDS" or "UTR" feature type without having to also filter out the "exon" regions
# that duplicate the "CDS" and "UTR" regions.
non_coding_transcript_exons = hl.bind(
lambda coding_transcripts: gene_exons.filter(
lambda exon: ~coding_transcripts.contains(exon.transcript_id)),
hl.set(
gene_exons.filter(lambda exon: (exon.feature_type == "CDS") |
(exon.feature_type == "UTR")).map(
lambda exon: exon.transcript_id)),
)
exons = (merge_overlapping_regions(
gene_exons.filter(lambda exon: exon.feature_type == "CDS")).extend(
merge_overlapping_regions(
gene_exons.filter(lambda exon: exon.feature_type == "UTR"))).
extend(merge_overlapping_regions(non_coding_transcript_exons)))
exons = exons.map(
lambda exon: exon.select("feature_type",
"start",
"stop",
xstart=xpos(exon.chrom, exon.start),
xstop=xpos(exon.chrom, exon.stop)))
return exons
def make_pop_filters_expr(ht: hl.Table,
qc_metrics: List[str]) -> hl.expr.SetExpression:
return hl.set(
hl.filter(lambda x: hl.is_defined(x), [
hl.or_missing(ht[f'fail_{metric}'], metric)
for metric in qc_metrics
]))
def make_hard_filters_expr(ht: hl.Table,
data_type: str) -> hl.expr.SetExpression:
"""
NOTE: additional metadata in Kristen's import file is hard-coded
:param: Table ht: input MT
:param: str data_type: 'exomes' or 'genomes'
:return: output MT
:rtype: SetExpression
"""
hard_filters = {
'contamination': ht.freemix > 0.05,
'callrate': ht.callrate < 0.85,
'chimera': ht.pct_chimeras > 0.05,
'ambiguous_sex': ht.ambiguous_sex
}
if data_type == 'exomes':
hard_filters.update({
'coverage': ht.mean_chr20_coverage == 0,
'sex_aneuploidy': ht.sex_aneuploidy
})
else:
hard_filters.update({
'coverage': ht.mean_dp < 15,
'insert_size': ht.median_insert_size < 250
})
return hl.set(
hl.filter(lambda x: hl.is_defined(x), [
hl.or_missing(filter_expr, name)
for name, filter_expr in hard_filters.items()
]))
def apply_mito_artifact_filter(
mt: hl.MatrixTable,
artifact_prone_sites_path: str,
) -> hl.MatrixTable:
"""Add back in artifact_prone_site filter
:param hl.MatrixTable mt: MatrixTable to use an input
:param str artifact_prone_sites_path: path to BED file of artifact_prone_sites to flag in the filters column
:return: MatrixTable with artifact_prone_sites filter
:rtype: hl.MatrixTable
"""
# apply "artifact_prone_site" filter to any SNP or deletion that spans a known problematic site
mt = mt.annotate_rows(
position_range=hl.range(mt.locus.position, mt.locus.position +
hl.len(mt.alleles[0])))
artifact_sites = []
with hl.hadoop_open(artifact_prone_sites_path) as f:
for line in f:
pos = line.split()[2]
artifact_sites.append(int(pos))
sites = hl.literal(set(artifact_sites))
mt = mt.annotate_rows(filters=hl.if_else(
hl.len(hl.set(mt.position_range).intersection(sites)) > 0,
{"artifact_prone_site"},
{"PASS"},
))
mt = mt.drop("position_range")
return mt
def remove_FT_values(
mt: hl.MatrixTable,
filters_to_remove: list = [
'possible_numt', 'mt_many_low_hets', 'FAIL', 'blacklisted_site'
]
) -> hl.MatrixTable:
"""Removes the FT filters specified in filters_to_remove
By default, this function removes the 'possible_numt', 'mt_many_low_hets', and 'FAIL' filters (because these filters were found to have low performance),
and the 'blacklisted_site' filter because this filter did not always behave as expected in early GATK versions (can be replaced with apply_mito_artifact_filter function)
:param hl.MatrixTable mt: MatrixTable
:param list filters_to_remove: list of FT filters that should be removed from the entries
:return: MatrixTable with certain FT filters removed
:rtype: MatrixTable
"""
filters_to_remove = hl.set(filters_to_remove)
mt = mt.annotate_entries(
FT=hl.array((mt.FT).difference(filters_to_remove)))
# if no filters exists after removing those specified above, set the FT field to PASS
mt = mt.annotate_entries(
FT=hl.if_else(hl.len(mt.FT) == 0, ["PASS"], mt.FT))
return (mt)
def combine(ts):
# pylint: disable=protected-access
tmp = ts.annotate(
alleles=merge_alleles(ts.data.map(lambda d: d.alleles)),
rsid=hl.find(hl.is_defined, ts.data.map(lambda d: d.rsid)),
filters=hl.set(hl.flatten(ts.data.map(lambda d: hl.array(d.filters)))),
info=hl.struct(
DP=hl.sum(ts.data.map(lambda d: d.info.DP)),
MQ_DP=hl.sum(ts.data.map(lambda d: d.info.MQ_DP)),
QUALapprox=hl.sum(ts.data.map(lambda d: d.info.QUALapprox)),
RAW_MQ=hl.sum(ts.data.map(lambda d: d.info.RAW_MQ)),
VarDP=hl.sum(ts.data.map(lambda d: d.info.VarDP)),
SB=hl.array([
hl.sum(ts.data.map(lambda d: d.info.SB[0])),
hl.sum(ts.data.map(lambda d: d.info.SB[1])),
hl.sum(ts.data.map(lambda d: d.info.SB[2])),
hl.sum(ts.data.map(lambda d: d.info.SB[3]))
])))
tmp = tmp.annotate(
__entries=hl.bind(
lambda combined_allele_index:
hl.range(0, hl.len(tmp.data)).flatmap(
lambda i:
hl.cond(hl.is_missing(tmp.data[i].__entries),
hl.range(0, hl.len(tmp.g[i].__cols))
.map(lambda _: hl.null(tmp.data[i].__entries.dtype.element_type)),
hl.bind(
lambda old_to_new: tmp.data[i].__entries.map(lambda e: renumber_entry(e, old_to_new)),
hl.range(0, hl.len(tmp.data[i].alleles)).map(
lambda j: combined_allele_index[tmp.data[i].alleles[j]])))),
hl.dict(hl.range(0, hl.len(tmp.alleles)).map(
lambda j: hl.tuple([tmp.alleles[j], j])))))
tmp = tmp.annotate_globals(__cols=hl.flatten(tmp.g.map(lambda g: g.__cols)))
return tmp.drop('data', 'g')
def add_filters_expr(
filters: Dict[str, hl.expr.BooleanExpression],
current_filters: hl.expr.SetExpression = None,
) -> hl.expr.SetExpression:
"""
Create an expression to create or add filters.
For each entry in the `filters` dictionary, if the value evaluates to `True`,
then the key is added as a filter name.
Current filters are kept if provided using `current_filters`
:param filters: The filters and their expressions
:param current_filters: The set of current filters
:return: An expression that can be used to annotate the filters
"""
if current_filters is None:
current_filters = hl.empty_set(hl.tstr)
return hl.fold(
lambda x, y: x.union(y),
current_filters,
[
hl.cond(filter_condition, hl.set([filter_name]),
hl.empty_set(hl.tstr))
for filter_name, filter_condition in filters.items()
],
)
def make_perm_filters_expr(ht: hl.Table,
data_type: str) -> hl.expr.SetExpression:
"""
NOTE: syndip will remain dropped wrt to permissions, but all possible QC measures will still be calculated
:param Table ht: input MT
:param str data_type: 'exomes' or 'genomes'
:return: output MT
:rtype: SetExpression
"""
if data_type == 'genomes':
perm_filters = {'not_releasable': ~ht.releasable_2_1}
else:
perm_filters = {
'tcga_tumor': ht.tcga_tumor,
'tcga_barcode': ht.tcga_weird_barcode,
'tcga_below_30': ht.tcga_below_30,
'specific_exclusion': ht.specific_exclusion,
'esp': ht.esp,
'not_releasable': ht.non_releasable,
'syndip': ht.syndip
}
return hl.set(
hl.filter(lambda x: hl.is_defined(x), [
hl.or_missing(filter_expr, name)
for name, filter_expr in perm_filters.items()
]))
def test_import_bgen_variant_filtering(self):
desired_variant_indexes = [1, 2, 3, 5, 7, 9, 11, 13, 17, 198]
actual = hl.import_bgen(resource('example.8bits.bgen'), ['GT'],
contig_recoding={'01': '1'},
reference_genome=None,
n_partitions=10,
_row_fields=['file_row_idx'],
_variants_per_file={
resource('example.8bits.bgen'):
desired_variant_indexes
})
# doing the expected import_bgen second catches the case where the
# hadoop configuraiton is polluted with old data from the
# _variants_per_file
everything = hl.import_bgen(resource('example.8bits.bgen'), ['GT'],
contig_recoding={'01': '1'},
reference_genome=None,
_row_fields=['file_row_idx'])
self.assertEqual(everything.count(), (199, 500))
expected = everything.filter_rows(
hl.set(desired_variant_indexes).contains(
hl.int32(everything.file_row_idx)))
self.assertTrue(expected._same(actual))
self.assertEqual(
(hl.str(actual.locus.contig) + ":" +
hl.str(actual.locus.position)).collect(), [
'1:3000', '1:4000', '1:5000', '1:7000', '1:9000', '1:11000',
'1:13000', '1:15000', '1:19000', '1:100001'
])
def vep_genes_expr(vep_expr: hl.expr.StructExpression,
least_consequence: str) -> hl.expr.SetExpression:
vep_consequences = hl.literal(
set(CSQ_ORDER[0:CSQ_ORDER.index(least_consequence) + 1]))
return (hl.set(
vep_expr.transcript_consequences.filter(
lambda tc: (tc.biotype == 'protein_coding') &
(tc.consequence_terms.any(lambda c: vep_consequences.contains(c))
)).map(lambda x: x.gene_id)))
def test_complex_round_trips():
assert_round_trip(hl.struct())
assert_round_trip(hl.empty_array(hl.tint32))
assert_round_trip(hl.empty_set(hl.tint32))
assert_round_trip(hl.empty_dict(hl.tint32, hl.tint32))
assert_round_trip(hl.locus('1', 100))
assert_round_trip(hl.struct(x=3))
assert_round_trip(hl.set([3, 4, 5, 3]))
assert_round_trip(hl.array([3, 4, 5]))
assert_round_trip(hl.dict({3: 'a', 4: 'b', 5: 'c'}))
assert_round_trip(
hl.struct(x=hl.dict({
3: 'a',
4: 'b',
5: 'c'
}),
y=hl.array([3, 4, 5]),
z=hl.set([3, 4, 5, 3])))
def get_expr_for_worst_transcript_consequence_annotations_struct(
vep_sorted_transcript_consequences_root,
include_coding_annotations=True):
"""Retrieves the top-ranked transcript annotation based on the ranking computed by
get_expr_for_vep_sorted_transcript_consequences_array(..)
Args:
vep_sorted_transcript_consequences_root (ArrayExpression):
include_coding_annotations (bool):
"""
transcript_consequences = {
"biotype": hl.tstr,
"canonical": hl.tint,
"category": hl.tstr,
"cdna_start": hl.tint,
"cdna_end": hl.tint,
"codons": hl.tstr,
"gene_id": hl.tstr,
"gene_symbol": hl.tstr,
"hgvs": hl.tstr,
"hgvsc": hl.tstr,
"major_consequence": hl.tstr,
"major_consequence_rank": hl.tint,
"transcript_id": hl.tstr,
}
if include_coding_annotations:
transcript_consequences.update({
"amino_acids": hl.tstr,
"domains": hl.tstr,
"hgvsp": hl.tstr,
"lof": hl.tstr,
"lof_flags": hl.tstr,
"lof_filter": hl.tstr,
"lof_info": hl.tstr,
"polyphen_prediction": hl.tstr,
"protein_id": hl.tstr,
"sift_prediction": hl.tstr,
})
return hl.cond(
vep_sorted_transcript_consequences_root.size() == 0,
hl.struct(
**{
field: hl.null(field_type)
for field, field_type in transcript_consequences.items()
}),
hl.bind(
lambda worst_transcript_consequence:
(worst_transcript_consequence.annotate(domains=hl.delimit(
hl.set(worst_transcript_consequence.domains))).select(
*transcript_consequences.keys())),
vep_sorted_transcript_consequences_root[0],
),
)
def make_filters_expr(ht: hl.Table,
qc_metrics: Iterable[str]) -> hl.expr.SetExpression:
return hl.set(
hl.filter(
lambda x: hl.is_defined(x),
[
hl.or_missing(ht[f"fail_{metric}"], metric)
for metric in qc_metrics
],
))
def vep_protein_domain_filter_expr(
d: hl.expr.DictExpression) -> hl.expr.BooleanExpression:
"""
Return True of False if any protein domain source(s) are contained within pre-defined protein domain sources.
Expected as input dict<k,v> where keys (k) represent source/database and values (v) the annotated domain_id.
:param d: hl.DictExpression
:return: hl.BoolExpression
"""
domain_dbs = hl.set(PROTEIN_DOMAIN_DB)
return (d.key_set().intersection(domain_dbs).length() >= 1)
def finalize_annotated_table_for_seqr_variants(
mt: hl.MatrixTable) -> hl.MatrixTable:
"""Given a messily-but-completely annotated Hail MatrixTable of variants,
return a new MatrixTable with appropriate formatting to export to Elasticsearch
and consume by Seqr.
TO-EXTREMELY-DO: Create a app/common Python 3 module with code for SeqrAnnotatedVariant,
with methods to im/export to/from Hail/Elasticsearch.
:param vep_mt: A VCF loaded into hail 0.2, VEP has been run,
and reference/computed fields have been added.
:type vep_mt: hl.MatrixTable
:return: A hail matrix table of variants and VEP annotations with
proper formatting to be consumed by Seqr.
:rtype: hl.MatrixTable
"""
mt = mt.annotate_rows(
sortedTranscriptConsequences=
get_expr_for_vep_sorted_transcript_consequences_array(vep_root=mt.vep))
mt = mt.annotate_rows(
mainTranscript=hl.cond(
hl.len(mt.sortedTranscriptConsequences) > 0,
mt.sortedTranscriptConsequences[0],
hl.null(
"struct {biotype: str,canonical: int32,cdna_start: int32,cdna_end: int32,codons: str,gene_id: str,gene_symbol: str,hgvsc: str,hgvsp: str,transcript_id: str,amino_acids: str,lof: str,lof_filter: str,lof_flags: str,lof_info: str,polyphen_prediction: str,protein_id: str,protein_start: int32,sift_prediction: str,consequence_terms: array<str>,domains: array<str>,major_consequence: str,category: str,hgvs: str,major_consequence_rank: int32,transcript_rank: int32}"
)),
#allele_id=clinvar_mt.index_rows(mt.row_key).vep.id,
alt=get_expr_for_alt_allele(mt),
chrom=get_expr_for_contig(mt.locus),
#clinvar_clinical_significance=clinvar_mt.index_rows(mt.row_key).clinical_significance,
domains=get_expr_for_vep_protein_domains_set(
vep_transcript_consequences_root=mt.vep.transcript_consequences),
geneIds=hl.set(
mt.vep.transcript_consequences.map(lambda c: c.gene_id)),
# gene_id_to_consequence_json=get_expr_for_vep_gene_id_to_consequence_map(
# vep_sorted_transcript_consequences_root=mt.sortedTranscriptConsequences,
# gene_ids=clinvar_mt.gene_ids
# ),
#gold_stars= clinvar_mt.index_entries(mt.row_key,mt.col_key).gold_stars,
pos=get_expr_for_start_pos(mt),
ref=get_expr_for_ref_allele(mt),
#review_status=clinvar_mt.index_rows(mt.locus,mt.alleles).review_status,
transcript_consequence_terms=get_expr_for_vep_consequence_terms_set(
vep_transcript_consequences_root=mt.sortedTranscriptConsequences),
transcript_ids=get_expr_for_vep_transcript_ids_set(
vep_transcript_consequences_root=mt.sortedTranscriptConsequences),
transcript_id_to_consequence_json=
get_expr_for_vep_transcript_id_to_consequence_map(
vep_transcript_consequences_root=mt.sortedTranscriptConsequences),
variant_id=get_expr_for_variant_id(mt),
xpos=get_expr_for_xpos(mt.locus))
return mt
def get_exons(gencode):
exons = gencode.filter(
hl.set(["exon", "CDS", "UTR"]).contains(gencode.feature))
exons = exons.select(
feature_type=exons.feature,
transcript_id=exons.transcript_id.split("\\.")[0],
gene_id=exons.gene_id.split("\\.")[0],
chrom=exons.interval.start.seqname[3:],
strand=exons.strand,
start=exons.interval.start.position,
stop=exons.interval.end.position,
)
return exons
def prepare_pext_data(base_level_pext_path, low_max_pext_genes_path):
ds = prepare_base_level_pext(base_level_pext_path)
low_max_pext_genes = hl.import_table(low_max_pext_genes_path)
low_max_pext_genes = low_max_pext_genes.aggregate(
hl.agg.collect_as_set(low_max_pext_genes.ensg))
ds = ds.annotate(flags=hl.if_else(
hl.set(low_max_pext_genes).contains(ds.gene_id),
hl.literal(["low_max_pext"]),
hl.empty_array(hl.tstr),
))
return ds
def main(args):
variants = hl.import_table(f'{ldprune_dir}/variants.txt', delimiter=' ')
variants = variants.annotate(alleles = hl.set([variants.ref, variants.alt]))
variants = variants.key_by('chrom','pos','alleles')
all_pops = [
'AFR',
'AMR',
'CSA',
'EAS',
'EUR',
'MID'
]
pops = [args.pop.upper()] if args.pop!=None else all_pops
for pop in pops:
bim0 = hl.import_table(f'{ldprune_dir}/subsets_5k/not_{pop}/not_{pop}.bim',
delimiter=' ',
no_header=True)
bim0 = bim0.rename({'f0':'chrom',
'f3':'pos',
'f4':'A1_bim',
'f5':'A2_bim'})
bim0 = bim0.add_index()
bim = bim0.annotate(alleles = hl.set([bim0.A1_bim, bim0.A2_bim]))
bim = bim.key_by('chrom','pos','alleles')
bim = bim.annotate(ref = variants[bim.key].ref,
alt = variants[bim.key].alt)
bim = bim.annotate(SNP = bim.chrom+':'+bim.pos+':'+bim.ref+':'+bim.alt)
bim = bim.key_by()
bim = bim.order_by(bim.idx)
bim = bim.select('chrom','SNP','f2','pos','A1_bim','A2_bim')
bim.export(f'{ldprune_dir}/subsets_5k/not_{pop}/not_{pop}.bim_new',
header=False,
delimiter=' ')
def main(args):
hl.init(master=f'local[{args.n_threads}]',
log=hl.utils.timestamp_path(os.path.join(tempfile.gettempdir(), 'extract_vcf'), suffix='.log'),
default_reference=args.reference)
sys.path.append('/')
add_args = []
if args.additional_args is not None:
add_args = args.additional_args.split(',')
load_module = importlib.import_module(args.load_module)
mt = getattr(load_module, args.load_mt_function)(*add_args)
if args.gene_map_ht_path is None:
interval = [hl.parse_locus_interval(args.interval)]
else:
gene_ht = hl.read_table(args.gene_map_ht_path)
if args.gene is not None:
gene_ht = gene_ht.filter(gene_ht.gene_symbol == args.gene)
interval = gene_ht.aggregate(hl.agg.take(gene_ht.interval, 1), _localize=False)
else:
interval = [hl.parse_locus_interval(args.interval)]
gene_ht = hl.filter_intervals(gene_ht, interval)
gene_ht = gene_ht.filter(hl.set(args.groups.split(',')).contains(gene_ht.annotation))
gene_ht.select(group=gene_ht.gene_id + '_' + gene_ht.gene_symbol + '_' + gene_ht.annotation, variant=hl.delimit(gene_ht.variants, '\t')
).key_by().drop('start').export(args.group_output_file, header=False)
# TODO: possible minor optimization: filter output VCF to only variants in `gene_ht.variants`
if not args.no_adj:
mt = mt.filter_entries(mt.adj)
mt = hl.filter_intervals(mt, interval)
if not args.input_bgen:
mt = mt.select_entries('GT')
mt = mt.filter_rows(hl.agg.count_where(mt.GT.is_non_ref()) > 0)
mt = mt.annotate_rows(rsid=mt.locus.contig + ':' + hl.str(mt.locus.position) + '_' + mt.alleles[0] + '/' + mt.alleles[1])
if args.callrate_filter:
mt = mt.filter_rows(hl.agg.fraction(hl.is_defined(mt.GT)) >= args.callrate_filter)
if args.export_bgen:
if not args.input_bgen:
mt = mt.annotate_entries(GT=hl.if_else(mt.GT.is_haploid(), hl.call(mt.GT[0], mt.GT[0]), mt.GT))
mt = gt_to_gp(mt)
mt = impute_missing_gp(mt, mean_impute=args.mean_impute_missing)
hl.export_bgen(mt, args.output_file, gp=mt.GP, varid=mt.rsid)
else:
mt = mt.annotate_entries(GT=hl.or_else(mt.GT, hl.call(0, 0)))
# Note: no mean-imputation for VCF
hl.export_vcf(mt, args.output_file)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("gencode")
parser.add_argument("canonical_transcripts")
parser.add_argument("hgnc")
parser.add_argument("--min-partitions", type=int, default=8)
parser.add_argument("--output", required=True)
args = parser.parse_args()
# Load genes from GTF file
genes = load_gencode_gene_models(args.gencode, min_partitions=args.min_partitions)
genes = genes.transmute(gencode_gene_symbol=genes.gene_symbol)
# Annotate genes with canonical transcript
canonical_transcripts = load_canonical_transcripts(args.canonical_transcripts, min_partitions=args.min_partitions)
genes = genes.annotate(canonical_transcript_id=canonical_transcripts[genes.gene_id].transcript_id)
# Drop transcripts except for canonical
genes = genes.annotate(
canonical_transcript=genes.transcripts.filter(
lambda transcript: transcript.transcript_id == genes.canonical_transcript_id
).head()
)
genes = genes.drop("transcripts")
# Annotate genes with information from HGNC
hgnc = load_hgnc(args.hgnc)
genes = genes.annotate(**hgnc[genes.gene_id])
genes = genes.annotate(symbol_source=hl.cond(hl.is_defined(genes.symbol), "hgnc", hl.null(hl.tstr)))
genes = genes.annotate(
symbol=hl.or_else(genes.symbol, genes.gencode_gene_symbol),
symbol_source=hl.or_else(genes.symbol_source, "gencode"),
)
# Collect all fields that can be used to search by gene symbol
genes = genes.annotate(
symbol_upper_case=genes.symbol.upper(),
search_terms=hl.set(
hl.empty_array(hl.tstr)
.append(genes.symbol)
.extend(genes.previous_symbols)
.extend(genes.alias_symbols)
.append(genes.gencode_gene_symbol)
.map(lambda s: s.upper())
),
)
genes.describe()
genes.write(args.output, overwrite=True)
def prepare_clinvar_variants(clinvar_path, reference_genome):
ds = hl.read_table(clinvar_path)
ds = ds.filter(hl.is_defined(ds[f"locus_{reference_genome}"]) & hl.is_defined(ds[f"alleles_{reference_genome}"]))
ds = ds.select(locus=ds[f"locus_{reference_genome}"], alleles=ds[f"alleles_{reference_genome}"], **ds.variant)
# Remove any variants with alleles other than ACGT
ds = ds.filter(
hl.len(hl.set(hl.delimit(ds.alleles, "").split("")).difference(hl.set(["A", "C", "G", "T", ""]))) == 0
)
ds = ds.annotate(
variant_id=variant_id(ds.locus, ds.alleles),
chrom=normalized_contig(ds.locus.contig),
pos=ds.locus.position,
ref=ds.alleles[0],
alt=ds.alleles[1],
)
ds = ds.key_by("locus", "alleles")
return ds
def import_hgnc(path):
ds = hl.import_table(path, missing="")
ds = ds.select(
hgnc_id=ds["HGNC ID"],
symbol=ds["Approved symbol"],
name=ds["Approved name"],
previous_symbols=ds["Previous symbols"],
alias_symbols=ds["Alias symbols"],
omim_id=ds["OMIM ID(supplied by OMIM)"],
gene_id=hl.or_else(ds["Ensembl gene ID"],
ds["Ensembl ID(supplied by Ensembl)"]),
)
ds = ds.filter(hl.is_defined(ds.gene_id)).key_by("gene_id")
ds = ds.annotate(
previous_symbols=hl.set(
ds.previous_symbols.split(",").map(lambda s: s.strip())),
alias_symbols=hl.set(
ds.alias_symbols.split(",").map(lambda s: s.strip())),
)
return ds
def test_import_bgen_variant_filtering_from_literals(self):
bgen_file = resource('example.8bits.bgen')
hl.index_bgen(bgen_file, contig_recoding={'01': '1'})
alleles = ['A', 'G']
desired_variants = [
hl.Struct(locus=hl.Locus('1', 2000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 2001), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 4000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 10000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 100001), alleles=alleles),
]
expected_result = [
hl.Struct(locus=hl.Locus('1', 2000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 2001), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 4000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 10000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 10000),
alleles=alleles), # Duplicated variant
hl.Struct(locus=hl.Locus('1', 100001), alleles=alleles),
]
part_1 = hl.import_bgen(
bgen_file,
['GT'],
n_partitions=1, # forcing seek to be called
variants=desired_variants)
self.assertTrue(part_1.rows().key_by(
'locus', 'alleles').select().collect() == expected_result)
part_199 = hl.import_bgen(
bgen_file,
['GT'],
n_partitions=
199, # forcing each variant to be its own partition for testing duplicates work properly
variants=desired_variants)
self.assertTrue(part_199.rows().key_by(
'locus', 'alleles').select().collect() == expected_result)
everything = hl.import_bgen(bgen_file, ['GT'])
self.assertEqual(everything.count(), (199, 500))
expected = everything.filter_rows(
hl.set(desired_variants).contains(everything.row_key))
self.assertTrue(expected._same(part_1))
def test_import_bgen_variant_filtering_from_literals(self):
bgen_file = resource('example.8bits.bgen')
hl.index_bgen(bgen_file,
contig_recoding={'01': '1'})
alleles = ['A', 'G']
desired_variants = [
hl.Struct(locus=hl.Locus('1', 2000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 2001), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 4000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 10000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 100001), alleles=alleles),
]
expected_result = [
hl.Struct(locus=hl.Locus('1', 2000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 2001), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 4000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 10000), alleles=alleles),
hl.Struct(locus=hl.Locus('1', 10000), alleles=alleles), # Duplicated variant
hl.Struct(locus=hl.Locus('1', 100001), alleles=alleles),
]
part_1 = hl.import_bgen(bgen_file,
['GT'],
n_partitions=1, # forcing seek to be called
variants=desired_variants)
self.assertTrue(part_1.rows().key_by('locus', 'alleles').select().collect() == expected_result)
part_199 = hl.import_bgen(bgen_file,
['GT'],
n_partitions=199, # forcing each variant to be its own partition for testing duplicates work properly
variants=desired_variants)
self.assertTrue(part_199.rows().key_by('locus', 'alleles').select().collect() == expected_result)
everything = hl.import_bgen(bgen_file, ['GT'])
self.assertEqual(everything.count(), (199, 500))
expected = everything.filter_rows(hl.set(desired_variants).contains(everything.row_key))
self.assertTrue(expected._same(part_1))
def create_all_values():
return hl.struct(
f32=hl.float32(3.14),
i64=hl.int64(-9),
m=hl.null(hl.tfloat64),
astruct=hl.struct(a=hl.null(hl.tint32), b=5.5),
mstruct=hl.null(hl.tstruct(x=hl.tint32, y=hl.tstr)),
aset=hl.set(['foo', 'bar', 'baz']),
mset=hl.null(hl.tset(hl.tfloat64)),
d=hl.dict({hl.array(['a', 'b']): 0.5, hl.array(['x', hl.null(hl.tstr), 'z']): 0.3}),
md=hl.null(hl.tdict(hl.tint32, hl.tstr)),
h38=hl.locus('chr22', 33878978, 'GRCh38'),
ml=hl.null(hl.tlocus('GRCh37')),
i=hl.interval(
hl.locus('1', 999),
hl.locus('1', 1001)),
c=hl.call(0, 1),
mc=hl.null(hl.tcall),
t=hl.tuple([hl.call(1, 2, phased=True), 'foo', hl.null(hl.tstr)]),
mt=hl.null(hl.ttuple(hl.tlocus('GRCh37'), hl.tbool))
)
def create_all_values_datasets():
all_values = hl.struct(
f32=hl.float32(3.14),
i64=hl.int64(-9),
m=hl.null(hl.tfloat64),
astruct=hl.struct(a=hl.null(hl.tint32), b=5.5),
mstruct=hl.null(hl.tstruct(x=hl.tint32, y=hl.tstr)),
aset=hl.set(['foo', 'bar', 'baz']),
mset=hl.null(hl.tset(hl.tfloat64)),
d=hl.dict({hl.array(['a', 'b']): 0.5, hl.array(['x', hl.null(hl.tstr), 'z']): 0.3}),
md=hl.null(hl.tdict(hl.tint32, hl.tstr)),
h38=hl.locus('chr22', 33878978, 'GRCh38'),
ml=hl.null(hl.tlocus('GRCh37')),
i=hl.interval(
hl.locus('1', 999),
hl.locus('1', 1001)),
c=hl.call(0, 1),
mc=hl.null(hl.tcall),
t=hl.tuple([hl.call(1, 2, phased=True), 'foo', hl.null(hl.tstr)]),
mt=hl.null(hl.ttuple(hl.tlocus('GRCh37'), hl.tbool))
)
def prefix(s, p):
return hl.struct(**{p + k: s[k] for k in s})
all_values_table = (hl.utils.range_table(5, n_partitions=3)
.annotate_globals(**prefix(all_values, 'global_'))
.annotate(**all_values)
.cache())
all_values_matrix_table = (hl.utils.range_matrix_table(3, 2, n_partitions=2)
.annotate_globals(**prefix(all_values, 'global_'))
.annotate_rows(**prefix(all_values, 'row_'))
.annotate_cols(**prefix(all_values, 'col_'))
.annotate_entries(**prefix(all_values, 'entry_'))
.cache())
return all_values_table, all_values_matrix_table
def ld_score_regression(weight_expr,
ld_score_expr,
chi_sq_exprs,
n_samples_exprs,
n_blocks=200,
two_step_threshold=30,
n_reference_panel_variants=None) -> Table:
r"""Estimate SNP-heritability and level of confounding biases from
GWAS summary statistics.
Given a set or multiple sets of genome-wide association study (GWAS)
summary statistics, :func:`.ld_score_regression` estimates the heritability
of a trait or set of traits and the level of confounding biases present in
the underlying studies by regressing chi-squared statistics on LD scores,
leveraging the model:
.. math::
\mathrm{E}[\chi_j^2] = 1 + Na + \frac{Nh_g^2}{M}l_j
* :math:`\mathrm{E}[\chi_j^2]` is the expected chi-squared statistic
for variant :math:`j` resulting from a test of association between
variant :math:`j` and a trait.
* :math:`l_j = \sum_{k} r_{jk}^2` is the LD score of variant
:math:`j`, calculated as the sum of squared correlation coefficients
between variant :math:`j` and nearby variants. See :func:`ld_score`
for further details.
* :math:`a` captures the contribution of confounding biases, such as
cryptic relatedness and uncontrolled population structure, to the
association test statistic.
* :math:`h_g^2` is the SNP-heritability, or the proportion of variation
in the trait explained by the effects of variants included in the
regression model above.
* :math:`M` is the number of variants used to estimate :math:`h_g^2`.
* :math:`N` is the number of samples in the underlying association study.
For more details on the method implemented in this function, see:
* `LD Score regression distinguishes confounding from polygenicity in genome-wide association studies (Bulik-Sullivan et al, 2015) <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4495769/>`__
Examples
--------
Run the method on a matrix table of summary statistics, where the rows
are variants and the columns are different phenotypes:
>>> mt_gwas = hl.read_matrix_table('data/ld_score_regression.sumstats.mt')
>>> ht_results = hl.experimental.ld_score_regression(
... weight_expr=mt_gwas['ld_score'],
... ld_score_expr=mt_gwas['ld_score'],
... chi_sq_exprs=mt_gwas['chi_squared'],
... n_samples_exprs=mt_gwas['n'])
Run the method on a table with summary statistics for a single
phenotype:
>>> ht_gwas = hl.read_table('data/ld_score_regression.sumstats.ht')
>>> ht_results = hl.experimental.ld_score_regression(
... weight_expr=ht_gwas['ld_score'],
... ld_score_expr=ht_gwas['ld_score'],
... chi_sq_exprs=ht_gwas['chi_squared_50_irnt'],
... n_samples_exprs=ht_gwas['n_50_irnt'])
Run the method on a table with summary statistics for multiple
phenotypes:
>>> ht_gwas = hl.read_table('data/ld_score_regression.sumstats.ht')
>>> ht_results = hl.experimental.ld_score_regression(
... weight_expr=ht_gwas['ld_score'],
... ld_score_expr=ht_gwas['ld_score'],
... chi_sq_exprs=[ht_gwas['chi_squared_50_irnt'],
... ht_gwas['chi_squared_20160']],
... n_samples_exprs=[ht_gwas['n_50_irnt'],
... ht_gwas['n_20160']])
Notes
-----
The ``exprs`` provided as arguments to :func:`.ld_score_regression`
must all be from the same object, either a :class:`Table` or a
:class:`MatrixTable`.
**If the arguments originate from a table:**
* The table must be keyed by fields ``locus`` of type
:class:`.tlocus` and ``alleles``, a :py:data:`.tarray` of
:py:data:`.tstr` elements.
* ``weight_expr``, ``ld_score_expr``, ``chi_sq_exprs``, and
``n_samples_exprs`` are must be row-indexed fields.
* The number of expressions passed to ``n_samples_exprs`` must be
equal to one or the number of expressions passed to
``chi_sq_exprs``. If just one expression is passed to
``n_samples_exprs``, that sample size expression is assumed to
apply to all sets of statistics passed to ``chi_sq_exprs``.
Otherwise, the expressions passed to ``chi_sq_exprs`` and
``n_samples_exprs`` are matched by index.
* The ``phenotype`` field that keys the table returned by
:func:`.ld_score_regression` will have generic :obj:`int` values
``0``, ``1``, etc. corresponding to the ``0th``, ``1st``, etc.
expressions passed to the ``chi_sq_exprs`` argument.
**If the arguments originate from a matrix table:**
* The dimensions of the matrix table must be variants
(rows) by phenotypes (columns).
* The rows of the matrix table must be keyed by fields
``locus`` of type :class:`.tlocus` and ``alleles``,
a :py:data:`.tarray` of :py:data:`.tstr` elements.
* The columns of the matrix table must be keyed by a field
of type :py:data:`.tstr` that uniquely identifies phenotypes
represented in the matrix table. The column key must be a single
expression; compound keys are not accepted.
* ``weight_expr`` and ``ld_score_expr`` must be row-indexed
fields.
* ``chi_sq_exprs`` must be a single entry-indexed field
(not a list of fields).
* ``n_samples_exprs`` must be a single entry-indexed field
(not a list of fields).
* The ``phenotype`` field that keys the table returned by
:func:`.ld_score_regression` will have values corresponding to the
column keys of the input matrix table.
This function returns a :class:`Table` with one row per set of summary
statistics passed to the ``chi_sq_exprs`` argument. The following
row-indexed fields are included in the table:
* **phenotype** (:py:data:`.tstr`) -- The name of the phenotype. The
returned table is keyed by this field. See the notes below for
details on the possible values of this field.
* **mean_chi_sq** (:py:data:`.tfloat64`) -- The mean chi-squared
test statistic for the given phenotype.
* **intercept** (`Struct`) -- Contains fields:
- **estimate** (:py:data:`.tfloat64`) -- A point estimate of the
intercept :math:`1 + Na`.
- **standard_error** (:py:data:`.tfloat64`) -- An estimate of
the standard error of this point estimate.
* **snp_heritability** (`Struct`) -- Contains fields:
- **estimate** (:py:data:`.tfloat64`) -- A point estimate of the
SNP-heritability :math:`h_g^2`.
- **standard_error** (:py:data:`.tfloat64`) -- An estimate of
the standard error of this point estimate.
Warning
-------
:func:`.ld_score_regression` considers only the rows for which both row
fields ``weight_expr`` and ``ld_score_expr`` are defined. Rows with missing
values in either field are removed prior to fitting the LD score
regression model.
Parameters
----------
weight_expr : :class:`.Float64Expression`
Row-indexed expression for the LD scores used to derive
variant weights in the model.
ld_score_expr : :class:`.Float64Expression`
Row-indexed expression for the LD scores used as covariates
in the model.
chi_sq_exprs : :class:`.Float64Expression` or :obj:`list` of
:class:`.Float64Expression`
One or more row-indexed (if table) or entry-indexed
(if matrix table) expressions for chi-squared
statistics resulting from genome-wide association
studies.
n_samples_exprs: :class:`.NumericExpression` or :obj:`list` of
:class:`.NumericExpression`
One or more row-indexed (if table) or entry-indexed
(if matrix table) expressions indicating the number of
samples used in the studies that generated the test
statistics supplied to ``chi_sq_exprs``.
n_blocks : :obj:`int`
The number of blocks used in the jackknife approach to
estimating standard errors.
two_step_threshold : :obj:`int`
Variants with chi-squared statistics greater than this
value are excluded in the first step of the two-step
procedure used to fit the model.
n_reference_panel_variants : :obj:`int`, optional
Number of variants used to estimate the
SNP-heritability :math:`h_g^2`.
Returns
-------
:class:`.Table`
Table keyed by ``phenotype`` with intercept and heritability estimates
for each phenotype passed to the function."""
chi_sq_exprs = wrap_to_list(chi_sq_exprs)
n_samples_exprs = wrap_to_list(n_samples_exprs)
assert ((len(chi_sq_exprs) == len(n_samples_exprs)) or
(len(n_samples_exprs) == 1))
__k = 2 # number of covariates, including intercept
ds = chi_sq_exprs[0]._indices.source
analyze('ld_score_regression/weight_expr',
weight_expr,
ds._row_indices)
analyze('ld_score_regression/ld_score_expr',
ld_score_expr,
ds._row_indices)
# format input dataset
if isinstance(ds, MatrixTable):
if len(chi_sq_exprs) != 1:
raise ValueError("""Only one chi_sq_expr allowed if originating
from a matrix table.""")
if len(n_samples_exprs) != 1:
raise ValueError("""Only one n_samples_expr allowed if
originating from a matrix table.""")
col_key = list(ds.col_key)
if len(col_key) != 1:
raise ValueError("""Matrix table must be keyed by a single
phenotype field.""")
analyze('ld_score_regression/chi_squared_expr',
chi_sq_exprs[0],
ds._entry_indices)
analyze('ld_score_regression/n_samples_expr',
n_samples_exprs[0],
ds._entry_indices)
ds = ds._select_all(row_exprs={'__locus': ds.locus,
'__alleles': ds.alleles,
'__w_initial': weight_expr,
'__w_initial_floor': hl.max(weight_expr,
1.0),
'__x': ld_score_expr,
'__x_floor': hl.max(ld_score_expr,
1.0)},
row_key=['__locus', '__alleles'],
col_exprs={'__y_name': ds[col_key[0]]},
col_key=['__y_name'],
entry_exprs={'__y': chi_sq_exprs[0],
'__n': n_samples_exprs[0]})
ds = ds.annotate_entries(**{'__w': ds.__w_initial})
ds = ds.filter_rows(hl.is_defined(ds.__locus) &
hl.is_defined(ds.__alleles) &
hl.is_defined(ds.__w_initial) &
hl.is_defined(ds.__x))
else:
assert isinstance(ds, Table)
for y in chi_sq_exprs:
analyze('ld_score_regression/chi_squared_expr', y, ds._row_indices)
for n in n_samples_exprs:
analyze('ld_score_regression/n_samples_expr', n, ds._row_indices)
ys = ['__y{:}'.format(i) for i, _ in enumerate(chi_sq_exprs)]
ws = ['__w{:}'.format(i) for i, _ in enumerate(chi_sq_exprs)]
ns = ['__n{:}'.format(i) for i, _ in enumerate(n_samples_exprs)]
ds = ds.select(**dict(**{'__locus': ds.locus,
'__alleles': ds.alleles,
'__w_initial': weight_expr,
'__x': ld_score_expr},
**{y: chi_sq_exprs[i]
for i, y in enumerate(ys)},
**{w: weight_expr for w in ws},
**{n: n_samples_exprs[i]
for i, n in enumerate(ns)}))
ds = ds.key_by(ds.__locus, ds.__alleles)
table_tmp_file = new_temp_file()
ds.write(table_tmp_file)
ds = hl.read_table(table_tmp_file)
hts = [ds.select(**{'__w_initial': ds.__w_initial,
'__w_initial_floor': hl.max(ds.__w_initial,
1.0),
'__x': ds.__x,
'__x_floor': hl.max(ds.__x, 1.0),
'__y_name': i,
'__y': ds[ys[i]],
'__w': ds[ws[i]],
'__n': hl.int(ds[ns[i]])})
for i, y in enumerate(ys)]
mts = [ht.to_matrix_table(row_key=['__locus',
'__alleles'],
col_key=['__y_name'],
row_fields=['__w_initial',
'__w_initial_floor',
'__x',
'__x_floor'])
for ht in hts]
ds = mts[0]
for i in range(1, len(ys)):
ds = ds.union_cols(mts[i])
ds = ds.filter_rows(hl.is_defined(ds.__locus) &
hl.is_defined(ds.__alleles) &
hl.is_defined(ds.__w_initial) &
hl.is_defined(ds.__x))
mt_tmp_file1 = new_temp_file()
ds.write(mt_tmp_file1)
mt = hl.read_matrix_table(mt_tmp_file1)
if not n_reference_panel_variants:
M = mt.count_rows()
else:
M = n_reference_panel_variants
# block variants for each phenotype
n_phenotypes = mt.count_cols()
mt = mt.annotate_entries(__in_step1=(hl.is_defined(mt.__y) &
(mt.__y < two_step_threshold)),
__in_step2=hl.is_defined(mt.__y))
mt = mt.annotate_cols(__col_idx=hl.int(hl.scan.count()),
__m_step1=hl.agg.count_where(mt.__in_step1),
__m_step2=hl.agg.count_where(mt.__in_step2))
col_keys = list(mt.col_key)
ht = mt.localize_entries(entries_array_field_name='__entries',
columns_array_field_name='__cols')
ht = ht.annotate(__entries=hl.rbind(
hl.scan.array_agg(
lambda entry: hl.scan.count_where(entry.__in_step1),
ht.__entries),
lambda step1_indices: hl.map(
lambda i: hl.rbind(
hl.int(hl.or_else(step1_indices[i], 0)),
ht.__cols[i].__m_step1,
ht.__entries[i],
lambda step1_idx, m_step1, entry: hl.rbind(
hl.map(
lambda j: hl.int(hl.floor(j * (m_step1 / n_blocks))),
hl.range(0, n_blocks + 1)),
lambda step1_separators: hl.rbind(
hl.set(step1_separators).contains(step1_idx),
hl.sum(
hl.map(
lambda s1: step1_idx >= s1,
step1_separators)) - 1,
lambda is_separator, step1_block: entry.annotate(
__step1_block=step1_block,
__step2_block=hl.cond(~entry.__in_step1 & is_separator,
step1_block - 1,
step1_block))))),
hl.range(0, hl.len(ht.__entries)))))
mt = ht._unlocalize_entries('__entries', '__cols', col_keys)
mt_tmp_file2 = new_temp_file()
mt.write(mt_tmp_file2)
mt = hl.read_matrix_table(mt_tmp_file2)
# initial coefficient estimates
mt = mt.annotate_cols(__initial_betas=[
1.0, (hl.agg.mean(mt.__y) - 1.0) / hl.agg.mean(mt.__x)])
mt = mt.annotate_cols(__step1_betas=mt.__initial_betas,
__step2_betas=mt.__initial_betas)
# step 1 iteratively reweighted least squares
for i in range(3):
mt = mt.annotate_entries(__w=hl.cond(
mt.__in_step1,
1.0/(mt.__w_initial_floor * 2.0 * (mt.__step1_betas[0] +
mt.__step1_betas[1] *
mt.__x_floor)**2),
0.0))
mt = mt.annotate_cols(__step1_betas=hl.agg.filter(
mt.__in_step1,
hl.agg.linreg(y=mt.__y,
x=[1.0, mt.__x],
weight=mt.__w).beta))
mt = mt.annotate_cols(__step1_h2=hl.max(hl.min(
mt.__step1_betas[1] * M / hl.agg.mean(mt.__n), 1.0), 0.0))
mt = mt.annotate_cols(__step1_betas=[
mt.__step1_betas[0],
mt.__step1_h2 * hl.agg.mean(mt.__n) / M])
# step 1 block jackknife
mt = mt.annotate_cols(__step1_block_betas=[
hl.agg.filter((mt.__step1_block != i) & mt.__in_step1,
hl.agg.linreg(y=mt.__y,
x=[1.0, mt.__x],
weight=mt.__w).beta)
for i in range(n_blocks)])
mt = mt.annotate_cols(__step1_block_betas_bias_corrected=hl.map(
lambda x: n_blocks * mt.__step1_betas - (n_blocks - 1) * x,
mt.__step1_block_betas))
mt = mt.annotate_cols(
__step1_jackknife_mean=hl.map(
lambda i: hl.mean(
hl.map(lambda x: x[i],
mt.__step1_block_betas_bias_corrected)),
hl.range(0, __k)),
__step1_jackknife_variance=hl.map(
lambda i: (hl.sum(
hl.map(lambda x: x[i]**2,
mt.__step1_block_betas_bias_corrected)) -
hl.sum(
hl.map(lambda x: x[i],
mt.__step1_block_betas_bias_corrected))**2 /
n_blocks) /
(n_blocks - 1) / n_blocks,
hl.range(0, __k)))
# step 2 iteratively reweighted least squares
for i in range(3):
mt = mt.annotate_entries(__w=hl.cond(
mt.__in_step2,
1.0/(mt.__w_initial_floor *
2.0 * (mt.__step2_betas[0] +
mt.__step2_betas[1] *
mt.__x_floor)**2),
0.0))
mt = mt.annotate_cols(__step2_betas=[
mt.__step1_betas[0],
hl.agg.filter(mt.__in_step2,
hl.agg.linreg(y=mt.__y - mt.__step1_betas[0],
x=[mt.__x],
weight=mt.__w).beta[0])])
mt = mt.annotate_cols(__step2_h2=hl.max(hl.min(
mt.__step2_betas[1] * M/hl.agg.mean(mt.__n), 1.0), 0.0))
mt = mt.annotate_cols(__step2_betas=[
mt.__step1_betas[0],
mt.__step2_h2 * hl.agg.mean(mt.__n)/M])
# step 2 block jackknife
mt = mt.annotate_cols(__step2_block_betas=[
hl.agg.filter((mt.__step2_block != i) & mt.__in_step2,
hl.agg.linreg(y=mt.__y - mt.__step1_betas[0],
x=[mt.__x],
weight=mt.__w).beta[0])
for i in range(n_blocks)])
mt = mt.annotate_cols(__step2_block_betas_bias_corrected=hl.map(
lambda x: n_blocks * mt.__step2_betas[1] - (n_blocks - 1) * x,
mt.__step2_block_betas))
mt = mt.annotate_cols(
__step2_jackknife_mean=hl.mean(
mt.__step2_block_betas_bias_corrected),
__step2_jackknife_variance=(
hl.sum(mt.__step2_block_betas_bias_corrected**2) -
hl.sum(mt.__step2_block_betas_bias_corrected)**2 /
n_blocks) / (n_blocks - 1) / n_blocks)
# combine step 1 and step 2 block jackknifes
mt = mt.annotate_entries(
__step2_initial_w=1.0/(mt.__w_initial_floor *
2.0 * (mt.__initial_betas[0] +
mt.__initial_betas[1] *
mt.__x_floor)**2))
mt = mt.annotate_cols(
__final_betas=[
mt.__step1_betas[0],
mt.__step2_betas[1]],
__c=(hl.agg.sum(mt.__step2_initial_w * mt.__x) /
hl.agg.sum(mt.__step2_initial_w * mt.__x**2)))
mt = mt.annotate_cols(__final_block_betas=hl.map(
lambda i: (mt.__step2_block_betas[i] - mt.__c *
(mt.__step1_block_betas[i][0] - mt.__final_betas[0])),
hl.range(0, n_blocks)))
mt = mt.annotate_cols(
__final_block_betas_bias_corrected=(n_blocks * mt.__final_betas[1] -
(n_blocks - 1) *
mt.__final_block_betas))
mt = mt.annotate_cols(
__final_jackknife_mean=[
mt.__step1_jackknife_mean[0],
hl.mean(mt.__final_block_betas_bias_corrected)],
__final_jackknife_variance=[
mt.__step1_jackknife_variance[0],
(hl.sum(mt.__final_block_betas_bias_corrected**2) -
hl.sum(mt.__final_block_betas_bias_corrected)**2 /
n_blocks) / (n_blocks - 1) / n_blocks])
# convert coefficient to heritability estimate
mt = mt.annotate_cols(
phenotype=mt.__y_name,
mean_chi_sq=hl.agg.mean(mt.__y),
intercept=hl.struct(
estimate=mt.__final_betas[0],
standard_error=hl.sqrt(mt.__final_jackknife_variance[0])),
snp_heritability=hl.struct(
estimate=(M/hl.agg.mean(mt.__n)) * mt.__final_betas[1],
standard_error=hl.sqrt((M/hl.agg.mean(mt.__n))**2 *
mt.__final_jackknife_variance[1])))
# format and return results
ht = mt.cols()
ht = ht.key_by(ht.phenotype)
ht = ht.select(ht.mean_chi_sq,
ht.intercept,
ht.snp_heritability)
ht_tmp_file = new_temp_file()
ht.write(ht_tmp_file)
ht = hl.read_table(ht_tmp_file)
return ht
def test_mendel_errors(self):
mt = hl.import_vcf(resource('mendel.vcf'))
ped = hl.Pedigree.read(resource('mendel.fam'))
men, fam, ind, var = hl.mendel_errors(mt['GT'], ped)
self.assertEqual(men.key.dtype, hl.tstruct(locus=mt.locus.dtype,
alleles=hl.tarray(hl.tstr),
s=hl.tstr))
self.assertEqual(men.row.dtype, hl.tstruct(locus=mt.locus.dtype,
alleles=hl.tarray(hl.tstr),
s=hl.tstr,
fam_id=hl.tstr,
mendel_code=hl.tint))
self.assertEqual(fam.key.dtype, hl.tstruct(pat_id=hl.tstr,
mat_id=hl.tstr))
self.assertEqual(fam.row.dtype, hl.tstruct(pat_id=hl.tstr,
mat_id=hl.tstr,
fam_id=hl.tstr,
children=hl.tint,
errors=hl.tint64,
snp_errors=hl.tint64))
self.assertEqual(ind.key.dtype, hl.tstruct(s=hl.tstr))
self.assertEqual(ind.row.dtype, hl.tstruct(s=hl.tstr,
fam_id=hl.tstr,
errors=hl.tint64,
snp_errors=hl.tint64))
self.assertEqual(var.key.dtype, hl.tstruct(locus=mt.locus.dtype,
alleles=hl.tarray(hl.tstr)))
self.assertEqual(var.row.dtype, hl.tstruct(locus=mt.locus.dtype,
alleles=hl.tarray(hl.tstr),
errors=hl.tint64))
self.assertEqual(men.count(), 41)
self.assertEqual(fam.count(), 2)
self.assertEqual(ind.count(), 7)
self.assertEqual(var.count(), mt.count_rows())
self.assertEqual(set(fam.select('children', 'errors', 'snp_errors').collect()),
{
hl.utils.Struct(pat_id='Dad1', mat_id='Mom1', children=2,
errors=41, snp_errors=39),
hl.utils.Struct(pat_id='Dad2', mat_id='Mom2', children=1,
errors=0, snp_errors=0)
})
self.assertEqual(set(ind.select('errors', 'snp_errors').collect()),
{
hl.utils.Struct(s='Son1', errors=23, snp_errors=22),
hl.utils.Struct(s='Dtr1', errors=18, snp_errors=17),
hl.utils.Struct(s='Dad1', errors=19, snp_errors=18),
hl.utils.Struct(s='Mom1', errors=22, snp_errors=21),
hl.utils.Struct(s='Dad2', errors=0, snp_errors=0),
hl.utils.Struct(s='Mom2', errors=0, snp_errors=0),
hl.utils.Struct(s='Son2', errors=0, snp_errors=0)
})
to_keep = hl.set([
(hl.Locus("1", 1), ['C', 'CT']),
(hl.Locus("1", 2), ['C', 'T']),
(hl.Locus("X", 1), ['C', 'T']),
(hl.Locus("X", 3), ['C', 'T']),
(hl.Locus("Y", 1), ['C', 'T']),
(hl.Locus("Y", 3), ['C', 'T'])
])
self.assertEqual(var.filter(to_keep.contains((var.locus, var.alleles)))
.order_by('locus')
.select('locus', 'alleles', 'errors').collect(),
[
hl.utils.Struct(locus=hl.Locus("1", 1), alleles=['C', 'CT'], errors=2),
hl.utils.Struct(locus=hl.Locus("1", 2), alleles=['C', 'T'], errors=1),
hl.utils.Struct(locus=hl.Locus("X", 1), alleles=['C', 'T'], errors=2),
hl.utils.Struct(locus=hl.Locus("X", 3), alleles=['C', 'T'], errors=1),
hl.utils.Struct(locus=hl.Locus("Y", 1), alleles=['C', 'T'], errors=1),
hl.utils.Struct(locus=hl.Locus("Y", 3), alleles=['C', 'T'], errors=1),
])
ped2 = hl.Pedigree.read(resource('mendelWithMissingSex.fam'))
men2, _, _, _ = hl.mendel_errors(mt['GT'], ped2)
self.assertTrue(men2.filter(men2.s == 'Dtr1')._same(men.filter(men.s == 'Dtr1')))
def merge_alleles(alleles) -> ArrayExpression:
return hl.array(hl.set(hl.flatten(alleles)))
