def test_annotate(self):
vds = self.get_vds()
vds = vds.annotate_globals(foo=5)
self.assertEqual(vds.globals.dtype, hl.tstruct(foo=hl.tint32))
vds = vds.annotate_rows(x1=agg.count(),
x2=agg.fraction(False),
x3=agg.count_where(True),
x4=vds.info.AC + vds.foo)
vds = vds.annotate_cols(apple=6)
vds = vds.annotate_cols(y1=agg.count(),
y2=agg.fraction(False),
y3=agg.count_where(True),
y4=vds.foo + vds.apple)
expected_schema = hl.tstruct(s=hl.tstr,
apple=hl.tint32,
y1=hl.tint64,
y2=hl.tfloat64,
y3=hl.tint64,
y4=hl.tint32)
self.assertTrue(
schema_eq(vds.col.dtype, expected_schema), "expected: " +
str(vds.col.dtype) + "\nactual: " + str(expected_schema))
vds = vds.select_entries(z1=vds.x1 + vds.foo,
z2=vds.x1 + vds.y1 + vds.foo)
self.assertTrue(
schema_eq(vds.entry.dtype, hl.tstruct(z1=hl.tint64, z2=hl.tint64)))
def test_annotate(self):
vds = self.get_vds()
vds = vds.annotate_globals(foo=5)
self.assertEqual(vds.globals.dtype, hl.tstruct(foo=hl.tint32))
vds = vds.annotate_rows(x1=agg.count(),
x2=agg.fraction(False),
x3=agg.count_where(True),
x4=vds.info.AC + vds.foo)
vds = vds.annotate_cols(apple=6)
vds = vds.annotate_cols(y1=agg.count(),
y2=agg.fraction(False),
y3=agg.count_where(True),
y4=vds.foo + vds.apple)
expected_schema = hl.tstruct(s=hl.tstr, apple=hl.tint32, y1=hl.tint64, y2=hl.tfloat64, y3=hl.tint64,
y4=hl.tint32)
self.assertTrue(schema_eq(vds.col.dtype, expected_schema),
"expected: " + str(vds.col.dtype) + "\nactual: " + str(expected_schema))
vds = vds.select_entries(z1=vds.x1 + vds.foo,
z2=vds.x1 + vds.y1 + vds.foo)
self.assertTrue(schema_eq(vds.entry.dtype, hl.tstruct(z1=hl.tint64, z2=hl.tint64)))
def hwe_normalize(call_expr):
mt = matrix_table_source('hwe_normalize/call_expr', call_expr)
mt = mt.select_entries(__gt=call_expr.n_alt_alleles())
mt = mt.annotate_rows(__AC=agg.sum(mt.__gt),
__n_called=agg.count_where(hl.is_defined(mt.__gt)))
mt = mt.filter_rows((mt.__AC > 0) & (mt.__AC < 2 * mt.__n_called))
n_variants = mt.count_rows()
if n_variants == 0:
raise FatalError(
"hwe_normalize: found 0 variants after filtering out monomorphic sites."
)
info(
f"hwe_normalize: found {n_variants} variants after filtering out monomorphic sites."
)
mt = mt.annotate_rows(__mean_gt=mt.__AC / mt.__n_called)
mt = mt.annotate_rows(__hwe_scaled_std_dev=hl.sqrt(mt.__mean_gt *
(2 - mt.__mean_gt) *
n_variants / 2))
mt = mt.unfilter_entries()
normalized_gt = hl.or_else(
(mt.__gt - mt.__mean_gt) / mt.__hwe_scaled_std_dev, 0.0)
return normalized_gt
def test_select_cols(self):
mt = hl.utils.range_matrix_table(3, 5, n_partitions=4)
mt = mt.annotate_entries(e=mt.col_idx * mt.row_idx)
mt = mt.annotate_globals(g=1)
mt = mt.annotate_cols(sum=agg.sum(mt.e + mt.col_idx + mt.row_idx + mt.g) + mt.col_idx + mt.g,
count=agg.count_where(mt.e % 2 == 0),
foo=agg.count())
result = convert_struct_to_dict(mt.cols().collect()[-2])
self.assertEqual(result, {'col_idx': 3, 'sum': 28, 'count': 2, 'foo': 3})
def test_select_cols(self):
mt = hl.utils.range_matrix_table(3, 5, n_partitions=4)
mt = mt.annotate_entries(e=mt.col_idx * mt.row_idx)
mt = mt.annotate_globals(g=1)
mt = mt.annotate_cols(sum=agg.sum(mt.e + mt.col_idx + mt.row_idx + mt.g) + mt.col_idx + mt.g,
count=agg.count_where(mt.e % 2 == 0),
foo=agg.count())
result = convert_struct_to_dict(mt.cols().collect()[-2])
self.assertEqual(result, {'col_idx': 3, 'sum': 28, 'count': 2, 'foo': 3})
def hwe_normalized_pca(dataset, k=10, compute_loadings=False, as_array=False):
"""Run principal component analysis (PCA) on the Hardy-Weinberg-normalized call matrix.
Examples
--------
>>> eigenvalues, scores, loadings = methods.hwe_normalized_pca(dataset, k=5)
Notes
-----
Variants that are all homozygous reference or all homozygous variant are removed before evaluation.
Parameters
----------
dataset : :class:`.MatrixTable`
Dataset.
k : :obj:`int`
Number of principal components.
compute_loadings : :obj:`bool`
If ``True``, compute row loadings.
as_array : :obj:`bool`
If ``True``, return scores and loadings as an array field. If ``False``, return
one field per element (`PC1`, `PC2`, ... `PCk`).
Returns
-------
(:obj:`list` of :obj:`float`, :class:`.Table`, :class:`.Table`)
List of eigenvalues, table with column scores, table with row loadings.
"""
dataset = dataset.annotate_rows(AC=agg.sum(dataset.GT.num_alt_alleles()),
n_called=agg.count_where(
functions.is_defined(dataset.GT)))
dataset = dataset.filter_rows(
(dataset.AC > 0) & (dataset.AC < 2 * dataset.n_called)).persist()
n_variants = dataset.count_rows()
if n_variants == 0:
raise FatalError(
"Cannot run PCA: found 0 variants after filtering out monomorphic sites."
)
info("Running PCA using {} variants.".format(n_variants))
entry_expr = functions.bind(
dataset.AC / dataset.n_called, lambda mean_gt: functions.cond(
functions.is_defined(dataset.GT), (dataset.GT.num_alt_alleles(
) - mean_gt) / functions.sqrt(mean_gt *
(2 - mean_gt) * n_variants / 2), 0))
result = pca(entry_expr, k, compute_loadings, as_array)
dataset.unpersist()
return result
def test_aggregate2(self):
schema = hl.tstruct(status=hl.tint32, GT=hl.tcall, qPheno=hl.tint32)
rows = [{'status': 0, 'GT': hl.Call([0, 0]), 'qPheno': 3},
{'status': 0, 'GT': hl.Call([0, 1]), 'qPheno': 13}]
kt = hl.Table.parallelize(rows, schema)
result = convert_struct_to_dict(
kt.group_by(status=kt.status)
.aggregate(
x1=agg.collect(kt.qPheno * 2),
x2=agg.explode(lambda elt: agg.collect(elt), [kt.qPheno, kt.qPheno + 1]),
x3=agg.min(kt.qPheno),
x4=agg.max(kt.qPheno),
x5=agg.sum(kt.qPheno),
x6=agg.product(hl.int64(kt.qPheno)),
x7=agg.count(),
x8=agg.count_where(kt.qPheno == 3),
x9=agg.fraction(kt.qPheno == 1),
x10=agg.stats(hl.float64(kt.qPheno)),
x11=agg.hardy_weinberg_test(kt.GT),
x13=agg.inbreeding(kt.GT, 0.1),
x14=agg.call_stats(kt.GT, ["A", "T"]),
x15=agg.collect(hl.Struct(a=5, b="foo", c=hl.Struct(banana='apple')))[0],
x16=agg.collect(hl.Struct(a=5, b="foo", c=hl.Struct(banana='apple')).c.banana)[0],
x17=agg.explode(lambda elt: agg.collect(elt), hl.null(hl.tarray(hl.tint32))),
x18=agg.explode(lambda elt: agg.collect(elt), hl.null(hl.tset(hl.tint32))),
x19=agg.take(kt.GT, 1, ordering=-kt.qPheno)
).take(1)[0])
expected = {u'status': 0,
u'x13': {u'n_called': 2, u'expected_homs': 1.64, u'f_stat': -1.777777777777777,
u'observed_homs': 1},
u'x14': {u'AC': [3, 1], u'AF': [0.75, 0.25], u'AN': 4, u'homozygote_count': [1, 0]},
u'x15': {u'a': 5, u'c': {u'banana': u'apple'}, u'b': u'foo'},
u'x10': {u'min': 3.0, u'max': 13.0, u'sum': 16.0, u'stdev': 5.0, u'n': 2, u'mean': 8.0},
u'x8': 1, u'x9': 0.0, u'x16': u'apple',
u'x11': {u'het_freq_hwe': 0.5, u'p_value': 0.5},
u'x2': [3, 4, 13, 14], u'x3': 3, u'x1': [6, 26], u'x6': 39, u'x7': 2, u'x4': 13, u'x5': 16,
u'x17': [],
u'x18': [],
u'x19': [hl.Call([0, 1])]}
self.maxDiff = None
self.assertDictEqual(result, expected)
def test_aggregate2(self):
schema = hl.tstruct(status=hl.tint32, GT=hl.tcall, qPheno=hl.tint32)
rows = [{'status': 0, 'GT': hl.Call([0, 0]), 'qPheno': 3},
{'status': 0, 'GT': hl.Call([0, 1]), 'qPheno': 13}]
kt = hl.Table.parallelize(rows, schema)
result = convert_struct_to_dict(
kt.group_by(status=kt.status)
.aggregate(
x1=agg.collect(kt.qPheno * 2),
x2=agg.explode(lambda elt: agg.collect(elt), [kt.qPheno, kt.qPheno + 1]),
x3=agg.min(kt.qPheno),
x4=agg.max(kt.qPheno),
x5=agg.sum(kt.qPheno),
x6=agg.product(hl.int64(kt.qPheno)),
x7=agg.count(),
x8=agg.count_where(kt.qPheno == 3),
x9=agg.fraction(kt.qPheno == 1),
x10=agg.stats(hl.float64(kt.qPheno)),
x11=agg.hardy_weinberg_test(kt.GT),
x13=agg.inbreeding(kt.GT, 0.1),
x14=agg.call_stats(kt.GT, ["A", "T"]),
x15=agg.collect(hl.Struct(a=5, b="foo", c=hl.Struct(banana='apple')))[0],
x16=agg.collect(hl.Struct(a=5, b="foo", c=hl.Struct(banana='apple')).c.banana)[0],
x17=agg.explode(lambda elt: agg.collect(elt), hl.null(hl.tarray(hl.tint32))),
x18=agg.explode(lambda elt: agg.collect(elt), hl.null(hl.tset(hl.tint32))),
x19=agg.take(kt.GT, 1, ordering=-kt.qPheno)
).take(1)[0])
expected = {u'status': 0,
u'x13': {u'n_called': 2, u'expected_homs': 1.64, u'f_stat': -1.777777777777777,
u'observed_homs': 1},
u'x14': {u'AC': [3, 1], u'AF': [0.75, 0.25], u'AN': 4, u'homozygote_count': [1, 0]},
u'x15': {u'a': 5, u'c': {u'banana': u'apple'}, u'b': u'foo'},
u'x10': {u'min': 3.0, u'max': 13.0, u'sum': 16.0, u'stdev': 5.0, u'n': 2, u'mean': 8.0},
u'x8': 1, u'x9': 0.0, u'x16': u'apple',
u'x11': {u'het_freq_hwe': 0.5, u'p_value': 0.5},
u'x2': [3, 4, 13, 14], u'x3': 3, u'x1': [6, 26], u'x6': 39, u'x7': 2, u'x4': 13, u'x5': 16,
u'x17': [],
u'x18': [],
u'x19': [hl.Call([0, 1])]}
self.maxDiff = None
self.assertDictEqual(result, expected)
def _make_tsm_from_call(call_expr,
block_size,
mean_center=False,
hwe_normalize=False):
mt = matrix_table_source('_make_tsm/entry_expr', call_expr)
mt = mt.select_entries(__gt=call_expr.n_alt_alleles())
if mean_center or hwe_normalize:
mt = mt.annotate_rows(__AC=agg.sum(mt.__gt),
__n_called=agg.count_where(hl.is_defined(
mt.__gt)))
mt = mt.filter_rows((mt.__AC > 0) & (mt.__AC < 2 * mt.__n_called))
n_variants = mt.count_rows()
if n_variants == 0:
raise FatalError(
"_make_tsm: found 0 variants after filtering out monomorphic sites."
)
info(
f"_make_tsm: found {n_variants} variants after filtering out monomorphic sites."
)
mt = mt.annotate_rows(__mean_gt=mt.__AC / mt.__n_called)
mt = mt.unfilter_entries()
mt = mt.select_entries(__x=hl.or_else(mt.__gt - mt.__mean_gt, 0.0))
if hwe_normalize:
mt = mt.annotate_rows(
__hwe_scaled_std_dev=hl.sqrt(mt.__mean_gt *
(2 - mt.__mean_gt) * n_variants /
2))
mt = mt.select_entries(__x=mt.__x / mt.__hwe_scaled_std_dev)
else:
mt = mt.select_entries(__x=mt.__gt)
A, ht = mt_to_table_of_ndarray(mt.__x,
block_size,
return_checkpointed_table_also=True)
A = A.persist()
return TallSkinnyMatrix(A, A.ndarray, ht, list(mt.col_key))
def pc_relate(call_expr,
min_individual_maf,
*,
k=None,
scores_expr=None,
min_kinship=None,
statistics="all",
block_size=None,
include_self_kinship=False) -> Table:
r"""Compute relatedness estimates between individuals using a variant of the
PC-Relate method.
.. include:: ../_templates/req_diploid_gt.rst
Examples
--------
Estimate kinship, identity-by-descent two, identity-by-descent one, and
identity-by-descent zero for every pair of samples, using a minimum minor
allele frequency filter of 0.01 and 10 principal components to control
for population structure.
>>> rel = hl.pc_relate(dataset.GT, 0.01, k=10)
Only compute the kinship statistic. This is more efficient than
computing all statistics.
>>> rel = hl.pc_relate(dataset.GT, 0.01, k=10, statistics='kin')
Compute all statistics, excluding sample-pairs with kinship less
than 0.1. This is more efficient than producing the full table and
then filtering using :meth:`.Table.filter`.
>>> rel = hl.pc_relate(dataset.GT, 0.01, k=10, min_kinship=0.1)
One can also pass in pre-computed principal component scores.
To produce the same results as in the previous example:
>>> _, scores_table, _ = hl.hwe_normalized_pca(dataset.GT,
... k=10,
... compute_loadings=False)
>>> rel = hl.pc_relate(dataset.GT,
... 0.01,
... scores_expr=scores_table[dataset.col_key].scores,
... min_kinship=0.1)
Notes
-----
The traditional estimator for kinship between a pair of individuals
:math:`i` and :math:`j`, sharing the set :math:`S_{ij}` of
single-nucleotide variants, from a population with allele frequencies
:math:`p_s`, is given by:
.. math::
\widehat{\phi_{ij}} \coloneqq
\frac{1}{|S_{ij}|}
\sum_{s \in S_{ij}}
\frac{(g_{is} - 2 p_s) (g_{js} - 2 p_s)}
{4 \sum_{s \in S_{ij}} p_s (1 - p_s)}
This estimator is true under the model that the sharing of common
(relative to the population) alleles is not very informative to
relatedness (because they're common) and the sharing of rare alleles
suggests a recent common ancestor from which the allele was inherited by
descent.
When multiple ancestry groups are mixed in a sample, this model breaks
down. Alleles that are rare in all but one ancestry group are treated as
very informative to relatedness. However, these alleles are simply
markers of the ancestry group. The PC-Relate method corrects for this
situation and the related situation of admixed individuals.
PC-Relate slightly modifies the usual estimator for relatedness:
occurrences of population allele frequency are replaced with an
"individual-specific allele frequency". This modification allows the
method to correctly weight an allele according to an individual's unique
ancestry profile.
The "individual-specific allele frequency" at a given genetic locus is
modeled by PC-Relate as a linear function of a sample's first ``k``
principal component coordinates. As such, the efficacy of this method
rests on two assumptions:
- an individual's first `k` principal component coordinates fully
describe their allele-frequency-relevant ancestry, and
- the relationship between ancestry (as described by principal
component coordinates) and population allele frequency is linear
The estimators for kinship, and identity-by-descent zero, one, and two
follow. Let:
- :math:`S_{ij}` be the set of genetic loci at which both individuals
:math:`i` and :math:`j` have a defined genotype
- :math:`g_{is} \in {0, 1, 2}` be the number of alternate alleles that
individual :math:`i` has at genetic locus :math:`s`
- :math:`\widehat{\mu_{is}} \in [0, 1]` be the individual-specific allele
frequency for individual :math:`i` at genetic locus :math:`s`
- :math:`{\widehat{\sigma^2_{is}}} \coloneqq \widehat{\mu_{is}} (1 - \widehat{\mu_{is}})`,
the binomial variance of :math:`\widehat{\mu_{is}}`
- :math:`\widehat{\sigma_{is}} \coloneqq \sqrt{\widehat{\sigma^2_{is}}}`,
the binomial standard deviation of :math:`\widehat{\mu_{is}}`
- :math:`\text{IBS}^{(0)}_{ij} \coloneqq \sum_{s \in S_{ij}} \mathbb{1}_{||g_{is} - g_{js} = 2||}`,
the number of genetic loci at which individuals :math:`i` and :math:`j`
share no alleles
- :math:`\widehat{f_i} \coloneqq 2 \widehat{\phi_{ii}} - 1`, the inbreeding
coefficient for individual :math:`i`
- :math:`g^D_{is}` be a dominance encoding of the genotype matrix, and
:math:`X_{is}` be a normalized dominance-coded genotype matrix
.. math::
g^D_{is} \coloneqq
\begin{cases}
\widehat{\mu_{is}} & g_{is} = 0 \\
0 & g_{is} = 1 \\
1 - \widehat{\mu_{is}} & g_{is} = 2
\end{cases}
\qquad
X_{is} \coloneqq g^D_{is} - \widehat{\sigma^2_{is}} (1 - \widehat{f_i})
The estimator for kinship is given by:
.. math::
\widehat{\phi_{ij}} \coloneqq
\frac{\sum_{s \in S_{ij}}(g - 2 \mu)_{is} (g - 2 \mu)_{js}}
{4 * \sum_{s \in S_{ij}}
\widehat{\sigma_{is}} \widehat{\sigma_{js}}}
The estimator for identity-by-descent two is given by:
.. math::
\widehat{k^{(2)}_{ij}} \coloneqq
\frac{\sum_{s \in S_{ij}}X_{is} X_{js}}{\sum_{s \in S_{ij}}
\widehat{\sigma^2_{is}} \widehat{\sigma^2_{js}}}
The estimator for identity-by-descent zero is given by:
.. math::
\widehat{k^{(0)}_{ij}} \coloneqq
\begin{cases}
\frac{\text{IBS}^{(0)}_{ij}}
{\sum_{s \in S_{ij}}
\widehat{\mu_{is}}^2(1 - \widehat{\mu_{js}})^2
+ (1 - \widehat{\mu_{is}})^2\widehat{\mu_{js}}^2}
& \widehat{\phi_{ij}} > 2^{-5/2} \\
1 - 4 \widehat{\phi_{ij}} + k^{(2)}_{ij}
& \widehat{\phi_{ij}} \le 2^{-5/2}
\end{cases}
The estimator for identity-by-descent one is given by:
.. math::
\widehat{k^{(1)}_{ij}} \coloneqq
1 - \widehat{k^{(2)}_{ij}} - \widehat{k^{(0)}_{ij}}
Note that, even if present, phase information is ignored by this method.
The PC-Relate method is described in "Model-free Estimation of Recent
Genetic Relatedness". Conomos MP, Reiner AP, Weir BS, Thornton TA. in
American Journal of Human Genetics. 2016 Jan 7. The reference
implementation is available in the `GENESIS Bioconductor package
<https://bioconductor.org/packages/release/bioc/html/GENESIS.html>`_ .
:func:`.pc_relate` differs from the reference implementation in a few
ways:
- if `k` is supplied, samples scores are computed via PCA on all samples,
not a specified subset of genetically unrelated samples. The latter
can be achieved by filtering samples, computing PCA variant loadings,
and using these loadings to compute and pass in scores for all samples.
- the estimators do not perform small sample correction
- the algorithm does not provide an option to use population-wide
allele frequency estimates
- the algorithm does not provide an option to not use "overall
standardization" (see R ``pcrelate`` documentation)
Under the PC-Relate model, kinship, :math:`\phi_{ij}`, ranges from 0 to
0.5, and is precisely half of the
fraction-of-genetic-material-shared. Listed below are the statistics for
a few pairings:
- Monozygotic twins share all their genetic material so their kinship
statistic is 0.5 in expection.
- Parent-child and sibling pairs both have kinship 0.25 in expectation
and are separated by the identity-by-descent-zero, :math:`k^{(2)}_{ij}`,
statistic which is zero for parent-child pairs and 0.25 for sibling
pairs.
- Avuncular pairs and grand-parent/-child pairs both have kinship 0.125
in expectation and both have identity-by-descent-zero 0.5 in expectation
- "Third degree relatives" are those pairs sharing
:math:`2^{-3} = 12.5 %` of their genetic material, the results of
PCRelate are often too noisy to reliably distinguish these pairs from
higher-degree-relative-pairs or unrelated pairs.
Note that :math:`g_{is}` is the number of alternate alleles. Hence, for
multi-allelic variants, a value of 2 may indicate two distinct alternative
alleles rather than a homozygous variant genotype. To enforce the latter,
either filter or split multi-allelic variants first.
The resulting table has the first 3, 4, 5, or 6 fields below, depending on
the `statistics` parameter:
- `i` (``col_key.dtype``) -- First sample. (key field)
- `j` (``col_key.dtype``) -- Second sample. (key field)
- `kin` (:py:data:`.tfloat64`) -- Kinship estimate, :math:`\widehat{\phi_{ij}}`.
- `ibd2` (:py:data:`.tfloat64`) -- IBD2 estimate, :math:`\widehat{k^{(2)}_{ij}}`.
- `ibd0` (:py:data:`.tfloat64`) -- IBD0 estimate, :math:`\widehat{k^{(0)}_{ij}}`.
- `ibd1` (:py:data:`.tfloat64`) -- IBD1 estimate, :math:`\widehat{k^{(1)}_{ij}}`.
Here ``col_key`` refers to the column key of the source matrix table,
and ``col_key.dtype`` is a struct containing the column key fields.
There is one row for each pair of distinct samples (columns), where `i`
corresponds to the column of smaller column index. In particular, if the
same column key value exists for :math:`n` columns, then the resulting
table will have :math:`\binom{n-1}{2}` rows with both key fields equal to
that column key value. This may result in unexpected behavior in downstream
processing.
Parameters
----------
call_expr : :class:`.CallExpression`
Entry-indexed call expression.
min_individual_maf : :obj:`float`
The minimum individual-specific minor allele frequency.
If either individual-specific minor allele frequency for a pair of
individuals is below this threshold, then the variant will not
be used to estimate relatedness for the pair.
k : :obj:`int`, optional
If set, `k` principal component scores are computed and used.
Exactly one of `k` and `scores_expr` must be specified.
scores_expr : :class:`.ArrayNumericExpression`, optional
Column-indexed expression of principal component scores, with the same
source as `call_expr`. All array values must have the same positive length,
corresponding to the number of principal components, and all scores must
be non-missing. Exactly one of `k` and `scores_expr` must be specified.
min_kinship : :obj:`float`, optional
If set, pairs of samples with kinship lower than `min_kinship` are excluded
from the results.
statistics : :class:`str`
Set of statistics to compute.
If ``'kin'``, only estimate the kinship statistic.
If ``'kin2'``, estimate the above and IBD2.
If ``'kin20'``, estimate the above and IBD0.
If ``'all'``, estimate the above and IBD1.
block_size : :obj:`int`, optional
Block size of block matrices used in the algorithm.
Default given by :meth:`.BlockMatrix.default_block_size`.
include_self_kinship: :obj:`bool`
If ``True``, include entries for an individual's estimated kinship with
themselves. Defaults to ``False``.
Returns
-------
:class:`.Table`
A :class:`.Table` mapping pairs of samples to their pair-wise statistics.
"""
mt = matrix_table_source('pc_relate/call_expr', call_expr)
if k and scores_expr is None:
_, scores, _ = hwe_normalized_pca(call_expr, k, compute_loadings=False)
scores_expr = scores[mt.col_key].scores
elif not k and scores_expr is not None:
analyze('pc_relate/scores_expr', scores_expr, mt._col_indices)
elif k and scores_expr is not None:
raise ValueError(
"pc_relate: exactly one of 'k' and 'scores_expr' must be set, found both"
)
else:
raise ValueError(
"pc_relate: exactly one of 'k' and 'scores_expr' must be set, found neither"
)
scores_table = mt.select_cols(__scores=scores_expr)\
.key_cols_by().select_cols('__scores').cols()
n_missing = scores_table.aggregate(
agg.count_where(hl.is_missing(scores_table.__scores)))
if n_missing > 0:
raise ValueError(
f'Found {n_missing} columns with missing scores array.')
mt = mt.select_entries(__gt=call_expr.n_alt_alleles()).unfilter_entries()
mt = mt.annotate_rows(__mean_gt=agg.mean(mt.__gt))
mean_imputed_gt = hl.or_else(hl.float64(mt.__gt), mt.__mean_gt)
if not block_size:
block_size = BlockMatrix.default_block_size()
g = BlockMatrix.from_entry_expr(mean_imputed_gt, block_size=block_size)
pcs = scores_table.collect(_localize=False).map(lambda x: x.__scores)
ht = Table(
ir.BlockMatrixToTableApply(
g._bmir, pcs._ir, {
'name': 'PCRelate',
'maf': min_individual_maf,
'blockSize': block_size,
'minKinship': min_kinship,
'statistics': {
'kin': 0,
'kin2': 1,
'kin20': 2,
'all': 3
}[statistics]
}))
if statistics == 'kin':
ht = ht.drop('ibd0', 'ibd1', 'ibd2')
elif statistics == 'kin2':
ht = ht.drop('ibd0', 'ibd1')
elif statistics == 'kin20':
ht = ht.drop('ibd1')
if not include_self_kinship:
ht = ht.filter(ht.i == ht.j, keep=False)
col_keys = hl.literal(mt.select_cols().key_cols_by().cols().collect(),
dtype=tarray(mt.col_key.dtype))
return ht.key_by(i=col_keys[ht.i], j=col_keys[ht.j])
def test_grm(self):
tolerance = 0.001
def load_id_file(path):
ids = []
with hl.hadoop_open(path) as f:
for l in f:
r = l.strip().split('\t')
self.assertEqual(len(r), 2)
ids.append(r[1])
return ids
def load_rel(ns, path):
rel = np.zeros((ns, ns))
with hl.hadoop_open(path) as f:
for i, l in enumerate(f):
for j, n in enumerate(map(float, l.strip().split('\t'))):
rel[i, j] = n
self.assertEqual(j, i)
self.assertEqual(i, ns - 1)
return rel
def load_grm(ns, nv, path):
m = np.zeros((ns, ns))
with utils.hadoop_open(path) as f:
i = 0
for l in f:
row = l.strip().split('\t')
self.assertEqual(int(row[2]), nv)
m[int(row[0]) - 1, int(row[1]) - 1] = float(row[3])
i += 1
self.assertEqual(i, ns * (ns + 1) / 2)
return m
def load_bin(ns, path):
m = np.zeros((ns, ns))
with utils.hadoop_open(path, 'rb') as f:
for i in range(ns):
for j in range(i + 1):
b = f.read(4)
self.assertEqual(len(b), 4)
m[i, j] = unpack('<f', bytearray(b))[0]
left = f.read()
self.assertEqual(len(left), 0)
return m
b_file = utils.new_temp_file(prefix="plink")
rel_file = utils.new_temp_file(prefix="test", suffix="rel")
rel_id_file = utils.new_temp_file(prefix="test", suffix="rel.id")
grm_file = utils.new_temp_file(prefix="test", suffix="grm")
grm_bin_file = utils.new_temp_file(prefix="test", suffix="grm.bin")
grm_nbin_file = utils.new_temp_file(prefix="test", suffix="grm.N.bin")
dataset = self.get_dataset()
n_samples = dataset.count_cols()
dataset = dataset.annotate_rows(AC=agg.sum(dataset.GT.n_alt_alleles()),
n_called=agg.count_where(hl.is_defined(dataset.GT)))
dataset = dataset.filter_rows((dataset.AC > 0) & (dataset.AC < 2 * dataset.n_called))
dataset = dataset.filter_rows(dataset.n_called == n_samples).persist()
hl.export_plink(dataset, b_file, id=dataset.s)
sample_ids = [row.s for row in dataset.cols().select('s').collect()]
n_variants = dataset.count_rows()
self.assertGreater(n_variants, 0)
grm = hl.genetic_relatedness_matrix(dataset)
grm.export_id_file(rel_id_file)
############
### rel
p_file = utils.new_temp_file(prefix="plink")
syscall('''plink --bfile {} --make-rel --out {}'''
.format(utils.uri_path(b_file), utils.uri_path(p_file)), shell=True, stdout=DEVNULL, stderr=DEVNULL)
self.assertEqual(load_id_file(p_file + ".rel.id"), sample_ids)
grm.export_rel(rel_file)
self.assertEqual(load_id_file(rel_id_file), sample_ids)
self.assertTrue(np.allclose(load_rel(n_samples, p_file + ".rel"),
load_rel(n_samples, rel_file),
atol=tolerance))
############
### gcta-grm
p_file = utils.new_temp_file(prefix="plink")
syscall('''plink --bfile {} --make-grm-gz --out {}'''
.format(utils.uri_path(b_file), utils.uri_path(p_file)), shell=True, stdout=DEVNULL, stderr=DEVNULL)
self.assertEqual(load_id_file(p_file + ".grm.id"), sample_ids)
grm.export_gcta_grm(grm_file)
self.assertTrue(np.allclose(load_grm(n_samples, n_variants, p_file + ".grm.gz"),
load_grm(n_samples, n_variants, grm_file),
atol=tolerance))
############
### gcta-grm-bin
p_file = utils.new_temp_file(prefix="plink")
syscall('''plink --bfile {} --make-grm-bin --out {}'''
.format(utils.uri_path(b_file), utils.uri_path(p_file)), shell=True, stdout=DEVNULL, stderr=DEVNULL)
self.assertEqual(load_id_file(p_file + ".grm.id"), sample_ids)
grm.export_gcta_grm_bin(grm_bin_file, grm_nbin_file)
self.assertTrue(np.allclose(load_bin(n_samples, p_file + ".grm.bin"),
load_bin(n_samples, grm_bin_file),
atol=tolerance))
self.assertTrue(np.allclose(load_bin(n_samples, p_file + ".grm.N.bin"),
load_bin(n_samples, grm_nbin_file),
atol=tolerance))
# COMMAND ----------
# MAGIC %md That's more like it! We may not be publishing ten new coffee-drinking loci in _Nature_, but we shouldn't expect to find anything but the strongest signals from a dataset of 284 individuals anyway.
# COMMAND ----------
# MAGIC %md ## Rare variant analysis
# MAGIC
# MAGIC Here we'll demonstrate how one can use the expression language to group and count by any arbitrary properties in row and column fields. Hail also implements the sequence kernel association test (SKAT).
# COMMAND ----------
entries = mt.entries()
results = (entries.group_by(pop=entries.SuperPopulation,
chromosome=entries.locus.contig).aggregate(
n_het=agg.count_where(entries.GT.is_het())))
# COMMAND ----------
results.show()
# COMMAND ----------
# MAGIC %md What if we want to group by minor allele frequency bin and hair color, and calculate the mean GQ?
# COMMAND ----------
entries = entries.annotate(
maf_bin=hl.cond(entries.info.AF[0] < 0.01, "< 1%",
hl.cond(entries.info.AF[0] < 0.05, "1%-5%", ">5%")))
def grm(dataset):
"""Compute the Genetic Relatedness Matrix (GRM).
.. include:: ../_templates/req_tvariant.rst
.. include:: ../_templates/req_biallelic.rst
Examples
--------
>>> km = methods.grm(dataset)
Notes
-----
The genetic relationship matrix (GRM) :math:`G` encodes genetic correlation
between each pair of samples. It is defined by :math:`G = MM^T` where
:math:`M` is a standardized version of the genotype matrix, computed as
follows. Let :math:`C` be the :math:`n \\times m` matrix of raw genotypes
in the variant dataset, with rows indexed by :math:`n` samples and columns
indexed by :math:`m` bialellic autosomal variants; :math:`C_{ij}` is the
number of alternate alleles of variant :math:`j` carried by sample
:math:`i`, which can be 0, 1, 2, or missing. For each variant :math:`j`,
the sample alternate allele frequency :math:`p_j` is computed as half the
mean of the non-missing entries of column :math:`j`. Entries of :math:`M`
are then mean-centered and variance-normalized as
.. math::
M_{ij} = \\frac{C_{ij}-2p_j}{\sqrt{2p_j(1-p_j)m}},
with :math:`M_{ij} = 0` for :math:`C_{ij}` missing (i.e. mean genotype
imputation). This scaling normalizes genotype variances to a common value
:math:`1/m` for variants in Hardy-Weinberg equilibrium and is further
motivated in the paper `Patterson, Price and Reich, 2006
<http://journals.plos.org/plosgenetics/article?id=10.1371/journal.pgen.0020190>`__.
(The resulting amplification of signal from the low end of the allele
frequency spectrum will also introduce noise for rare variants; common
practice is to filter out variants with minor allele frequency below some
cutoff.) The factor :math:`1/m` gives each sample row approximately unit
total variance (assuming linkage equilibrium) so that the diagonal entries
of the GRM are approximately 1. Equivalently,
.. math::
G_{ik} = \\frac{1}{m} \\sum_{j=1}^m \\frac{(C_{ij}-2p_j)(C_{kj}-2p_j)}{2 p_j (1-p_j)}
Warning
-------
Since Hardy-Weinberg normalization cannot be applied to variants that
contain only reference alleles or only alternate alleles, all such variants
are removed prior to calcularing the GRM.
Parameters
----------
dataset : :class:`.MatrixTable`
Dataset to sample from.
Returns
-------
:class:`genetics.KinshipMatrix`
Genetic Relatedness Matrix for all samples.
:rtype:
"""
dataset = dataset.annotate_rows(AC=agg.sum(dataset.GT.num_alt_alleles()),
n_called=agg.count_where(
functions.is_defined(dataset.GT)))
dataset = dataset.filter_rows(
(dataset.AC > 0) & (dataset.AC < 2 * dataset.n_called)).persist()
n_variants = dataset.count_rows()
if n_variants == 0:
raise FatalError(
"Cannot run GRM: found 0 variants after filtering out monomorphic sites."
)
info("Computing GRM using {} variants.".format(n_variants))
normalized_genotype_expr = functions.bind(
dataset.AC / dataset.n_called, lambda mean_gt: functions.cond(
functions.is_defined(dataset.GT), (dataset.GT.num_alt_alleles(
) - mean_gt) / functions.sqrt(mean_gt *
(2 - mean_gt) * n_variants / 2), 0))
bm = BlockMatrix.from_matrix_table(normalized_genotype_expr)
dataset.unpersist()
grm = bm.T.dot(bm)
return KinshipMatrix._from_block_matrix(
dataset.colkey_schema, grm,
[row.s
for row in dataset.cols_table().select('s').collect()], n_variants)
