def test_from_entry_expr_options(self): def build_mt(a): data = [{'v': 0, 's': 0, 'x': a[0]}, {'v': 0, 's': 1, 'x': a[1]}, {'v': 0, 's': 2, 'x': a[2]}] ht = hl.Table.parallelize(data, hl.dtype('struct{v: int32, s: int32, x: float64}')) mt = ht.to_matrix_table(['v'], ['s']) ids = mt.key_cols_by()['s'].collect() return mt.choose_cols([ids.index(0), ids.index(1), ids.index(2)]) def check(expr, mean_impute, center, normalize, expected): actual = np.squeeze(BlockMatrix.from_entry_expr(expr, mean_impute=mean_impute, center=center, normalize=normalize).to_numpy()) assert np.allclose(actual, expected) a = np.array([0.0, 1.0, 2.0]) mt = build_mt(a) check(mt.x, False, False, False, a) check(mt.x, False, True, False, a - 1.0) check(mt.x, False, False, True, a / np.sqrt(5)) check(mt.x, False, True, True, (a - 1.0) / np.sqrt(2)) check(mt.x + 1 - 1, False, False, False, a) mt = build_mt([0.0, hl.null('float64'), 2.0]) check(mt.x, True, False, False, a) check(mt.x, True, True, False, a - 1.0) check(mt.x, True, False, True, a / np.sqrt(5)) check(mt.x, True, True, True, (a - 1.0) / np.sqrt(2)) with self.assertRaises(Exception): BlockMatrix.from_entry_expr(mt.x)
def test_from_entry_expr_options(self): def build_mt(a): data = [{'v': 0, 's': 0, 'x': a[0]}, {'v': 0, 's': 1, 'x': a[1]}, {'v': 0, 's': 2, 'x': a[2]}] ht = hl.Table.parallelize(data, hl.dtype('struct{v: int32, s: int32, x: float64}')) mt = ht.to_matrix_table(['v'], ['s']) ids = mt.key_cols_by()['s'].collect() return mt.choose_cols([ids.index(0), ids.index(1), ids.index(2)]) def check(expr, mean_impute, center, normalize, expected): actual = np.squeeze(BlockMatrix.from_entry_expr(expr, mean_impute=mean_impute, center=center, normalize=normalize).to_numpy()) assert np.allclose(actual, expected) a = np.array([0.0, 1.0, 2.0]) mt = build_mt(a) check(mt.x, False, False, False, a) check(mt.x, False, True, False, a - 1.0) check(mt.x, False, False, True, a / np.sqrt(5)) check(mt.x, False, True, True, (a - 1.0) / np.sqrt(2)) check(mt.x + 1 - 1, False, False, False, a) mt = build_mt([0.0, hl.null('float64'), 2.0]) check(mt.x, True, False, False, a) check(mt.x, True, True, False, a - 1.0) check(mt.x, True, False, True, a / np.sqrt(5)) check(mt.x, True, True, True, (a - 1.0) / np.sqrt(2)) with self.assertRaises(Exception): BlockMatrix.from_entry_expr(mt.x)
def test_from_entry_expr(self): mt = get_dataset() mt = mt.annotate_entries(x=hl.or_else(mt.GT.n_alt_alleles(), 0)).cache() a1 = BlockMatrix.from_entry_expr(hl.or_else(mt.GT.n_alt_alleles(), 0), block_size=32).to_numpy() a2 = BlockMatrix.from_entry_expr(mt.x, block_size=32).to_numpy() a3 = BlockMatrix.from_entry_expr(hl.float64(mt.x), block_size=32).to_numpy() self._assert_eq(a1, a2) self._assert_eq(a1, a3) path = new_temp_file() BlockMatrix.write_from_entry_expr(mt.x, path, block_size=32) a4 = BlockMatrix.read(path).to_numpy() self._assert_eq(a1, a4)
def test_from_entry_expr(self): mt = get_dataset() mt = mt.annotate_entries(x=hl.or_else(mt.GT.n_alt_alleles(), 0)).cache() a1 = BlockMatrix.from_entry_expr(hl.or_else(mt.GT.n_alt_alleles(), 0), block_size=32).to_numpy() a2 = BlockMatrix.from_entry_expr(mt.x, block_size=32).to_numpy() a3 = BlockMatrix.from_entry_expr(hl.float64(mt.x), block_size=32).to_numpy() self._assert_eq(a1, a2) self._assert_eq(a1, a3) with hl.TemporaryDirectory(ensure_exists=False) as path: BlockMatrix.write_from_entry_expr(mt.x, path, block_size=32) a4 = BlockMatrix.read(path).to_numpy() self._assert_eq(a1, a4)
def test_from_entry_expr(self): mt = get_dataset() mt = mt.annotate_entries(x=hl.or_else(mt.GT.n_alt_alleles(), 0)).cache() a1 = BlockMatrix.from_entry_expr(hl.or_else(mt.GT.n_alt_alleles(), 0), block_size=32).to_numpy() a2 = BlockMatrix.from_entry_expr(mt.x, block_size=32).to_numpy() a3 = BlockMatrix.from_entry_expr(hl.float64(mt.x), block_size=32).to_numpy() self._assert_eq(a1, a2) self._assert_eq(a1, a3) path = new_temp_file() BlockMatrix.write_from_entry_expr(mt.x, path, block_size=32) a4 = BlockMatrix.read(path).to_numpy() self._assert_eq(a1, a4)
def check(expr, mean_impute, center, normalize, expected): actual = np.squeeze( BlockMatrix.from_entry_expr(expr, mean_impute=mean_impute, center=center, normalize=normalize).to_numpy()) assert np.allclose(actual, expected)
def test_to_matrix_table(self): n_partitions = 2 rows, cols = 2, 5 bm = BlockMatrix._create(rows, cols, [float(i) for i in range(10)]) actual = bm.to_matrix_table_row_major(n_partitions) expected = hl.utils.range_matrix_table(rows, cols) expected = expected.annotate_entries(element=hl.float64(expected.row_idx * cols + expected.col_idx)) expected = expected.key_cols_by(col_idx=hl.int64(expected.col_idx)) expected = expected.key_rows_by(row_idx=hl.int64(expected.row_idx)) assert expected._same(actual) bm = BlockMatrix.random(50, 100, block_size=25, seed=0) mt = bm.to_matrix_table_row_major(n_partitions) mt_round_trip = BlockMatrix.from_entry_expr(mt.element).to_matrix_table_row_major() assert mt._same(mt_round_trip)
def test_to_matrix_table(self): n_partitions = 2 rows, cols = 2, 5 bm = BlockMatrix._create(rows, cols, [float(i) for i in range(10)]) actual = bm.to_matrix_table_row_major(n_partitions) expected = hl.utils.range_matrix_table(rows, cols) expected = expected.annotate_entries(element=hl.float64(expected.row_idx * cols + expected.col_idx)) expected = expected.key_cols_by(col_idx=hl.int64(expected.col_idx)) expected = expected.key_rows_by(row_idx=hl.int64(expected.row_idx)) assert expected._same(actual) bm = BlockMatrix.random(50, 100, block_size=25, seed=0) mt = bm.to_matrix_table_row_major(n_partitions) mt_round_trip = BlockMatrix.from_entry_expr(mt.element).to_matrix_table_row_major() assert mt._same(mt_round_trip)
def test_write_from_entry_expr_overwrite(self): mt = hl.balding_nichols_model(1, 1, 1) mt = mt.select_entries(x=mt.GT.n_alt_alleles()) bm = BlockMatrix.from_entry_expr(mt.x) path = new_temp_file() BlockMatrix.write_from_entry_expr(mt.x, path) self.assertRaises(FatalError, lambda: BlockMatrix.write_from_entry_expr(mt.x, path)) BlockMatrix.write_from_entry_expr(mt.x, path, overwrite=True) self._assert_eq(BlockMatrix.read(path), bm) # non-field expressions currently take a separate code path path2 = new_temp_file() BlockMatrix.write_from_entry_expr(mt.x + 1, path2) self.assertRaises(FatalError, lambda: BlockMatrix.write_from_entry_expr(mt.x + 1, path2)) BlockMatrix.write_from_entry_expr(mt.x + 2, path2, overwrite=True) self._assert_eq(BlockMatrix.read(path2), bm + 2)
def test_write_from_entry_expr_overwrite(self): mt = hl.balding_nichols_model(1, 1, 1) mt = mt.select_entries(x=mt.GT.n_alt_alleles()) bm = BlockMatrix.from_entry_expr(mt.x) with hl.TemporaryDirectory(ensure_exists=False) as path: BlockMatrix.write_from_entry_expr(mt.x, path) self.assertRaises(FatalError, lambda: BlockMatrix.write_from_entry_expr(mt.x, path)) BlockMatrix.write_from_entry_expr(mt.x, path, overwrite=True) self._assert_eq(BlockMatrix.read(path), bm) with hl.TemporaryDirectory(ensure_exists=False) as path: # non-field expressions currently take a separate code path BlockMatrix.write_from_entry_expr(mt.x + 1, path) self.assertRaises(FatalError, lambda: BlockMatrix.write_from_entry_expr(mt.x + 1, path)) BlockMatrix.write_from_entry_expr(mt.x + 2, path, overwrite=True) self._assert_eq(BlockMatrix.read(path), bm + 2)
def test_write_from_entry_expr_overwrite(self): mt = hl.balding_nichols_model(1, 1, 1) mt = mt.select_entries(x=mt.GT.n_alt_alleles()) bm = BlockMatrix.from_entry_expr(mt.x) path = new_temp_file() BlockMatrix.write_from_entry_expr(mt.x, path) self.assertRaises(FatalError, lambda: BlockMatrix.write_from_entry_expr(mt.x, path)) BlockMatrix.write_from_entry_expr(mt.x, path, overwrite=True) self._assert_eq(BlockMatrix.read(path), bm) # non-field expressions currently take a separate code path path2 = new_temp_file() BlockMatrix.write_from_entry_expr(mt.x + 1, path2) self.assertRaises(FatalError, lambda: BlockMatrix.write_from_entry_expr(mt.x + 1, path2)) BlockMatrix.write_from_entry_expr(mt.x + 2, path2, overwrite=True) self._assert_eq(BlockMatrix.read(path2), bm + 2)
def direct_calculation(ds): ds = BlockMatrix.from_entry_expr(ds['GT'].n_alt_alleles()).to_numpy() # filter out constant rows isconst = lambda r: any([all([(gt < c + .01) and (gt > c - .01) for gt in r]) for c in range(3)]) ds = np.array([row for row in ds if not isconst(row)]) nvariants, nsamples = ds.shape sumgt = lambda r: sum([i for i in r if i >= 0]) sumsq = lambda r: sum([i ** 2 for i in r if i >= 0]) mean = [sumgt(row) / nsamples for row in ds] stddev = [sqrt(sumsq(row) / nsamples - mean[i] ** 2) for i, row in enumerate(ds)] mat = np.array([[(g - mean[i]) / stddev[i] for g in row] for i, row in enumerate(ds)]) rrm = (mat.T @ mat) / nvariants return rrm
def get_ref_X(ref_panel, overwrite=False): r''' Returns N_ref x M dim matrix of column-standardized genotypes of LD ref panel ''' X_bm_path = f'{bucket}/{ref_panel}.X.bm' if overwrite or not hl.hadoop_is_file(f'{X_bm_path}/_SUCCESS'): mt = hl.import_plink(bed=f'{bucket}/{ref_panel}.bed', bim=f'{bucket}/{ref_panel}.bim', fam=f'{bucket}/{ref_panel}.fam') mt = mt.annotate_rows(stats=hl.agg.stats(mt.GT.n_alt_alleles())) mt = mt.annotate_entries(X=(mt.GT.n_alt_alleles() - mt.stats.mean) / mt.stats.stdev) X = BlockMatrix.from_entry_expr(mt.X) X = X.T X.write(f'{bucket}/{ref_panel}.X.bm', overwrite=True) X = BlockMatrix.read(X_bm_path) return X
def ld_score(entry_expr, locus_expr, radius, coord_expr=None, annotation_exprs=None, block_size=None) -> Table: """Calculate LD scores. Example ------- >>> # Load genetic data into MatrixTable >>> mt = hl.import_plink(bed='data/ldsc.bed', ... bim='data/ldsc.bim', ... fam='data/ldsc.fam') >>> # Create locus-keyed Table with numeric variant annotations >>> ht = hl.import_table('data/ldsc.annot', ... types={'BP': hl.tint, ... 'binary': hl.tfloat, ... 'continuous': hl.tfloat}) >>> ht = ht.annotate(locus=hl.locus(ht.CHR, ht.BP)) >>> ht = ht.key_by('locus') >>> # Annotate MatrixTable with external annotations >>> mt = mt.annotate_rows(binary_annotation=ht[mt.locus].binary, ... continuous_annotation=ht[mt.locus].continuous) >>> # Calculate LD scores using centimorgan coordinates >>> ht_scores = hl.experimental.ld_score(entry_expr=mt.GT.n_alt_alleles(), ... locus_expr=mt.locus, ... radius=1.0, ... coord_expr=mt.cm_position, ... annotation_exprs=[mt.binary_annotation, ... mt.continuous_annotation]) >>> # Show results >>> ht_scores.show(3) .. code-block:: text +---------------+-------------------+-----------------------+-------------+ | locus | binary_annotation | continuous_annotation | univariate | +---------------+-------------------+-----------------------+-------------+ | locus<GRCh37> | float64 | float64 | float64 | +---------------+-------------------+-----------------------+-------------+ | 20:82079 | 1.15183e+00 | 7.30145e+01 | 1.60117e+00 | | 20:103517 | 2.04604e+00 | 2.75392e+02 | 4.69239e+00 | | 20:108286 | 2.06585e+00 | 2.86453e+02 | 5.00124e+00 | +---------------+-------------------+-----------------------+-------------+ Warning ------- :func:`.ld_score` will fail if ``entry_expr`` results in any missing values. The special float value ``nan`` is not considered a missing value. **Further reading** For more in-depth discussion of LD scores, see: - `LD Score regression distinguishes confounding from polygenicity in genome-wide association studies (Bulik-Sullivan et al, 2015) <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4495769/>`__ - `Partitioning heritability by functional annotation using genome-wide association summary statistics (Finucane et al, 2015) <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4626285/>`__ Notes ----- `entry_expr`, `locus_expr`, `coord_expr` (if specified), and `annotation_exprs` (if specified) must come from the same MatrixTable. Parameters ---------- entry_expr : :class:`.NumericExpression` Expression for entries of genotype matrix (e.g. ``mt.GT.n_alt_alleles()``). locus_expr : :class:`.LocusExpression` Row-indexed locus expression. radius : :obj:`int` or :obj:`float` Radius of window for row values (in units of `coord_expr` if set, otherwise in units of basepairs). coord_expr: :class:`.Float64Expression`, optional Row-indexed numeric expression for the row value used to window variants. By default, the row value is given by the locus position. annotation_exprs : :class:`.NumericExpression` or :obj:`list` of :class:`.NumericExpression`, optional Annotation expression(s) to partition LD scores. Univariate annotation will always be included and does not need to be specified. block_size : :obj:`int`, optional Block size. Default given by :meth:`.BlockMatrix.default_block_size`. Returns ------- :class:`.Table` Table keyed by `locus_expr` with LD scores for each variant and `annotation_expr`. The function will always return LD scores for the univariate (all SNPs) annotation.""" mt = entry_expr._indices.source mt_locus_expr = locus_expr._indices.source if coord_expr is None: mt_coord_expr = mt_locus_expr else: mt_coord_expr = coord_expr._indices.source if not annotation_exprs: check_mts = all([mt == mt_locus_expr, mt == mt_coord_expr]) else: check_mts = all( [mt == mt_locus_expr, mt == mt_coord_expr] + [mt == x._indices.source for x in wrap_to_list(annotation_exprs)]) if not check_mts: raise ValueError("""ld_score: entry_expr, locus_expr, coord_expr (if specified), and annotation_exprs (if specified) must come from same MatrixTable.""") n = mt.count_cols() r2 = hl.row_correlation(entry_expr, block_size)**2 r2_adj = ((n - 1.0) / (n - 2.0)) * r2 - (1.0 / (n - 2.0)) starts, stops = hl.linalg.utils.locus_windows(locus_expr, radius, coord_expr) r2_adj_sparse = r2_adj.sparsify_row_intervals(starts, stops) r2_adj_sparse_tmp = new_temp_file() r2_adj_sparse.write(r2_adj_sparse_tmp) r2_adj_sparse = BlockMatrix.read(r2_adj_sparse_tmp) if not annotation_exprs: cols = ['univariate'] col_idxs = {0: 'univariate'} l2 = r2_adj_sparse.sum(axis=1) else: ht = mt.select_rows(*wrap_to_list(annotation_exprs)).rows() ht = ht.annotate(univariate=hl.literal(1.0)) names = [name for name in ht.row if name not in ht.key] ht_union = hl.Table.union(*[(ht.annotate( name=hl.str(x), value=hl.float(ht[x])).select('name', 'value')) for x in names]) mt_annotations = ht_union.to_matrix_table(row_key=list(ht_union.key), col_key=['name']) cols = mt_annotations['name'].collect() col_idxs = {i: cols[i] for i in range(len(cols))} a = BlockMatrix.from_entry_expr(mt_annotations.value) a_tmp = new_temp_file() BlockMatrix.write_from_entry_expr(mt_annotations.value, a_tmp) a = BlockMatrix.read(a_tmp) l2 = r2_adj_sparse @ a l2_bm_tmp = new_temp_file() l2_tsv_tmp = new_temp_file() l2.write(l2_bm_tmp, force_row_major=True) BlockMatrix.export(l2_bm_tmp, l2_tsv_tmp) ht_scores = hl.import_table(l2_tsv_tmp, no_header=True, impute=True) ht_scores = ht_scores.add_index() ht_scores = ht_scores.key_by('idx') ht_scores = ht_scores.rename( {'f{:}'.format(i): col_idxs[i] for i in range(len(cols))}) ht = mt.select_rows(__locus=locus_expr).rows() ht = ht.add_index() ht = ht.annotate(**ht_scores[ht.idx]) ht = ht.key_by('__locus') ht = ht.select(*[x for x in ht_scores.row if x not in ht_scores.key]) ht = ht.rename({'__locus': 'locus'}) return ht
def pc_relate(call_expr, min_individual_maf, *, k=None, scores_expr=None, min_kinship=None, statistics="all", block_size=None, include_self_kinship=False) -> Table: r"""Compute relatedness estimates between individuals using a variant of the PC-Relate method. .. include:: ../_templates/req_diploid_gt.rst Examples -------- Estimate kinship, identity-by-descent two, identity-by-descent one, and identity-by-descent zero for every pair of samples, using a minimum minor allele frequency filter of 0.01 and 10 principal components to control for population structure. >>> rel = hl.pc_relate(dataset.GT, 0.01, k=10) Only compute the kinship statistic. This is more efficient than computing all statistics. >>> rel = hl.pc_relate(dataset.GT, 0.01, k=10, statistics='kin') Compute all statistics, excluding sample-pairs with kinship less than 0.1. This is more efficient than producing the full table and then filtering using :meth:`.Table.filter`. >>> rel = hl.pc_relate(dataset.GT, 0.01, k=10, min_kinship=0.1) One can also pass in pre-computed principal component scores. To produce the same results as in the previous example: >>> _, scores_table, _ = hl.hwe_normalized_pca(dataset.GT, ... k=10, ... compute_loadings=False) >>> rel = hl.pc_relate(dataset.GT, ... 0.01, ... scores_expr=scores_table[dataset.col_key].scores, ... min_kinship=0.1) Notes ----- The traditional estimator for kinship between a pair of individuals :math:`i` and :math:`j`, sharing the set :math:`S_{ij}` of single-nucleotide variants, from a population with allele frequencies :math:`p_s`, is given by: .. math:: \widehat{\phi_{ij}} \coloneqq \frac{1}{|S_{ij}|} \sum_{s \in S_{ij}} \frac{(g_{is} - 2 p_s) (g_{js} - 2 p_s)} {4 \sum_{s \in S_{ij}} p_s (1 - p_s)} This estimator is true under the model that the sharing of common (relative to the population) alleles is not very informative to relatedness (because they're common) and the sharing of rare alleles suggests a recent common ancestor from which the allele was inherited by descent. When multiple ancestry groups are mixed in a sample, this model breaks down. Alleles that are rare in all but one ancestry group are treated as very informative to relatedness. However, these alleles are simply markers of the ancestry group. The PC-Relate method corrects for this situation and the related situation of admixed individuals. PC-Relate slightly modifies the usual estimator for relatedness: occurrences of population allele frequency are replaced with an "individual-specific allele frequency". This modification allows the method to correctly weight an allele according to an individual's unique ancestry profile. The "individual-specific allele frequency" at a given genetic locus is modeled by PC-Relate as a linear function of a sample's first ``k`` principal component coordinates. As such, the efficacy of this method rests on two assumptions: - an individual's first `k` principal component coordinates fully describe their allele-frequency-relevant ancestry, and - the relationship between ancestry (as described by principal component coordinates) and population allele frequency is linear The estimators for kinship, and identity-by-descent zero, one, and two follow. Let: - :math:`S_{ij}` be the set of genetic loci at which both individuals :math:`i` and :math:`j` have a defined genotype - :math:`g_{is} \in {0, 1, 2}` be the number of alternate alleles that individual :math:`i` has at genetic locus :math:`s` - :math:`\widehat{\mu_{is}} \in [0, 1]` be the individual-specific allele frequency for individual :math:`i` at genetic locus :math:`s` - :math:`{\widehat{\sigma^2_{is}}} \coloneqq \widehat{\mu_{is}} (1 - \widehat{\mu_{is}})`, the binomial variance of :math:`\widehat{\mu_{is}}` - :math:`\widehat{\sigma_{is}} \coloneqq \sqrt{\widehat{\sigma^2_{is}}}`, the binomial standard deviation of :math:`\widehat{\mu_{is}}` - :math:`\text{IBS}^{(0)}_{ij} \coloneqq \sum_{s \in S_{ij}} \mathbb{1}_{||g_{is} - g_{js} = 2||}`, the number of genetic loci at which individuals :math:`i` and :math:`j` share no alleles - :math:`\widehat{f_i} \coloneqq 2 \widehat{\phi_{ii}} - 1`, the inbreeding coefficient for individual :math:`i` - :math:`g^D_{is}` be a dominance encoding of the genotype matrix, and :math:`X_{is}` be a normalized dominance-coded genotype matrix .. math:: g^D_{is} \coloneqq \begin{cases} \widehat{\mu_{is}} & g_{is} = 0 \\ 0 & g_{is} = 1 \\ 1 - \widehat{\mu_{is}} & g_{is} = 2 \end{cases} \qquad X_{is} \coloneqq g^D_{is} - \widehat{\sigma^2_{is}} (1 - \widehat{f_i}) The estimator for kinship is given by: .. math:: \widehat{\phi_{ij}} \coloneqq \frac{\sum_{s \in S_{ij}}(g - 2 \mu)_{is} (g - 2 \mu)_{js}} {4 * \sum_{s \in S_{ij}} \widehat{\sigma_{is}} \widehat{\sigma_{js}}} The estimator for identity-by-descent two is given by: .. math:: \widehat{k^{(2)}_{ij}} \coloneqq \frac{\sum_{s \in S_{ij}}X_{is} X_{js}}{\sum_{s \in S_{ij}} \widehat{\sigma^2_{is}} \widehat{\sigma^2_{js}}} The estimator for identity-by-descent zero is given by: .. math:: \widehat{k^{(0)}_{ij}} \coloneqq \begin{cases} \frac{\text{IBS}^{(0)}_{ij}} {\sum_{s \in S_{ij}} \widehat{\mu_{is}}^2(1 - \widehat{\mu_{js}})^2 + (1 - \widehat{\mu_{is}})^2\widehat{\mu_{js}}^2} & \widehat{\phi_{ij}} > 2^{-5/2} \\ 1 - 4 \widehat{\phi_{ij}} + k^{(2)}_{ij} & \widehat{\phi_{ij}} \le 2^{-5/2} \end{cases} The estimator for identity-by-descent one is given by: .. math:: \widehat{k^{(1)}_{ij}} \coloneqq 1 - \widehat{k^{(2)}_{ij}} - \widehat{k^{(0)}_{ij}} Note that, even if present, phase information is ignored by this method. The PC-Relate method is described in "Model-free Estimation of Recent Genetic Relatedness". Conomos MP, Reiner AP, Weir BS, Thornton TA. in American Journal of Human Genetics. 2016 Jan 7. The reference implementation is available in the `GENESIS Bioconductor package <https://bioconductor.org/packages/release/bioc/html/GENESIS.html>`_ . :func:`.pc_relate` differs from the reference implementation in a few ways: - if `k` is supplied, samples scores are computed via PCA on all samples, not a specified subset of genetically unrelated samples. The latter can be achieved by filtering samples, computing PCA variant loadings, and using these loadings to compute and pass in scores for all samples. - the estimators do not perform small sample correction - the algorithm does not provide an option to use population-wide allele frequency estimates - the algorithm does not provide an option to not use "overall standardization" (see R ``pcrelate`` documentation) Under the PC-Relate model, kinship, :math:`\phi_{ij}`, ranges from 0 to 0.5, and is precisely half of the fraction-of-genetic-material-shared. Listed below are the statistics for a few pairings: - Monozygotic twins share all their genetic material so their kinship statistic is 0.5 in expection. - Parent-child and sibling pairs both have kinship 0.25 in expectation and are separated by the identity-by-descent-zero, :math:`k^{(2)}_{ij}`, statistic which is zero for parent-child pairs and 0.25 for sibling pairs. - Avuncular pairs and grand-parent/-child pairs both have kinship 0.125 in expectation and both have identity-by-descent-zero 0.5 in expectation - "Third degree relatives" are those pairs sharing :math:`2^{-3} = 12.5 %` of their genetic material, the results of PCRelate are often too noisy to reliably distinguish these pairs from higher-degree-relative-pairs or unrelated pairs. Note that :math:`g_{is}` is the number of alternate alleles. Hence, for multi-allelic variants, a value of 2 may indicate two distinct alternative alleles rather than a homozygous variant genotype. To enforce the latter, either filter or split multi-allelic variants first. The resulting table has the first 3, 4, 5, or 6 fields below, depending on the `statistics` parameter: - `i` (``col_key.dtype``) -- First sample. (key field) - `j` (``col_key.dtype``) -- Second sample. (key field) - `kin` (:py:data:`.tfloat64`) -- Kinship estimate, :math:`\widehat{\phi_{ij}}`. - `ibd2` (:py:data:`.tfloat64`) -- IBD2 estimate, :math:`\widehat{k^{(2)}_{ij}}`. - `ibd0` (:py:data:`.tfloat64`) -- IBD0 estimate, :math:`\widehat{k^{(0)}_{ij}}`. - `ibd1` (:py:data:`.tfloat64`) -- IBD1 estimate, :math:`\widehat{k^{(1)}_{ij}}`. Here ``col_key`` refers to the column key of the source matrix table, and ``col_key.dtype`` is a struct containing the column key fields. There is one row for each pair of distinct samples (columns), where `i` corresponds to the column of smaller column index. In particular, if the same column key value exists for :math:`n` columns, then the resulting table will have :math:`\binom{n-1}{2}` rows with both key fields equal to that column key value. This may result in unexpected behavior in downstream processing. Parameters ---------- call_expr : :class:`.CallExpression` Entry-indexed call expression. min_individual_maf : :obj:`float` The minimum individual-specific minor allele frequency. If either individual-specific minor allele frequency for a pair of individuals is below this threshold, then the variant will not be used to estimate relatedness for the pair. k : :obj:`int`, optional If set, `k` principal component scores are computed and used. Exactly one of `k` and `scores_expr` must be specified. scores_expr : :class:`.ArrayNumericExpression`, optional Column-indexed expression of principal component scores, with the same source as `call_expr`. All array values must have the same positive length, corresponding to the number of principal components, and all scores must be non-missing. Exactly one of `k` and `scores_expr` must be specified. min_kinship : :obj:`float`, optional If set, pairs of samples with kinship lower than `min_kinship` are excluded from the results. statistics : :class:`str` Set of statistics to compute. If ``'kin'``, only estimate the kinship statistic. If ``'kin2'``, estimate the above and IBD2. If ``'kin20'``, estimate the above and IBD0. If ``'all'``, estimate the above and IBD1. block_size : :obj:`int`, optional Block size of block matrices used in the algorithm. Default given by :meth:`.BlockMatrix.default_block_size`. include_self_kinship: :obj:`bool` If ``True``, include entries for an individual's estimated kinship with themselves. Defaults to ``False``. Returns ------- :class:`.Table` A :class:`.Table` mapping pairs of samples to their pair-wise statistics. """ mt = matrix_table_source('pc_relate/call_expr', call_expr) if k and scores_expr is None: _, scores, _ = hwe_normalized_pca(call_expr, k, compute_loadings=False) scores_expr = scores[mt.col_key].scores elif not k and scores_expr is not None: analyze('pc_relate/scores_expr', scores_expr, mt._col_indices) elif k and scores_expr is not None: raise ValueError( "pc_relate: exactly one of 'k' and 'scores_expr' must be set, found both" ) else: raise ValueError( "pc_relate: exactly one of 'k' and 'scores_expr' must be set, found neither" ) scores_table = mt.select_cols(__scores=scores_expr)\ .key_cols_by().select_cols('__scores').cols() n_missing = scores_table.aggregate( agg.count_where(hl.is_missing(scores_table.__scores))) if n_missing > 0: raise ValueError( f'Found {n_missing} columns with missing scores array.') mt = mt.select_entries(__gt=call_expr.n_alt_alleles()).unfilter_entries() mt = mt.annotate_rows(__mean_gt=agg.mean(mt.__gt)) mean_imputed_gt = hl.or_else(hl.float64(mt.__gt), mt.__mean_gt) if not block_size: block_size = BlockMatrix.default_block_size() g = BlockMatrix.from_entry_expr(mean_imputed_gt, block_size=block_size) pcs = scores_table.collect(_localize=False).map(lambda x: x.__scores) ht = Table( ir.BlockMatrixToTableApply( g._bmir, pcs._ir, { 'name': 'PCRelate', 'maf': min_individual_maf, 'blockSize': block_size, 'minKinship': min_kinship, 'statistics': { 'kin': 0, 'kin2': 1, 'kin20': 2, 'all': 3 }[statistics] })) if statistics == 'kin': ht = ht.drop('ibd0', 'ibd1', 'ibd2') elif statistics == 'kin2': ht = ht.drop('ibd0', 'ibd1') elif statistics == 'kin20': ht = ht.drop('ibd1') if not include_self_kinship: ht = ht.filter(ht.i == ht.j, keep=False) col_keys = hl.literal(mt.select_cols().key_cols_by().cols().collect(), dtype=tarray(mt.col_key.dtype)) return ht.key_by(i=col_keys[ht.i], j=col_keys[ht.j])
def check(expr, mean_impute, center, normalize, expected): actual = np.squeeze(BlockMatrix.from_entry_expr(expr, mean_impute=mean_impute, center=center, normalize=normalize).to_numpy()) assert np.allclose(actual, expected)
def test_linear_mixed_model_fastlmm(self): # FastLMM Test data is from all.bed, all.bim, all.fam, cov.txt, pheno_10_causals.txt: # https://github.com/MicrosoftGenomics/FaST-LMM/tree/master/tests/datasets/synth # # Data is filtered to chromosome 1,3 and samples 0-124,375-499 (2000 variants and 250 samples) # # Results are computed with single_snp (with LOCO) as in: # https://github.com/MicrosoftGenomics/FaST-LMM/blob/master/doc/ipynb/FaST-LMM.ipynb n, m = 250, 1000 # per chromosome x_table = hl.import_table(resource('fastlmmCov.txt'), no_header=True, impute=True).key_by('f1') y_table = hl.import_table(resource('fastlmmPheno.txt'), no_header=True, impute=True, delimiter=' ').key_by('f1') mt = hl.import_plink(bed=resource('fastlmmTest.bed'), bim=resource('fastlmmTest.bim'), fam=resource('fastlmmTest.fam'), reference_genome=None) mt = mt.annotate_cols(x=x_table[mt.col_key].f2) mt = mt.annotate_cols(y=y_table[mt.col_key].f2).cache() x = np.array([np.ones(n), mt.x.collect()]).T y = np.array(mt.y.collect()) mt_chr1 = mt.filter_rows(mt.locus.contig == '1') mt_chr3 = mt.filter_rows(mt.locus.contig == '3') # testing chrom 1 for h2, betas, p-values h2_fastlmm = 0.14276125 beta_fastlmm = [0.012202061, 0.037718282, -0.033572693, 0.29171541, -0.045644170] # FastLMM p-values do not agree to high precision because FastLMM regresses # out x from each SNP first and does an F(1, dof)-test on (beta / se)^2 # (t-test), whereas Hail does likelihood ratio test. # We verify below that Hail's p-values remain fixed going forward. # fastlmm = [0.84650294, 0.57865098, 0.59050998, 1.6649473e-06, 0.46892059] pval_hail = [0.84543084, 0.57596760, 0.58788517, 1.4057279e-06, 0.46578204] gamma_fastlmm = h2_fastlmm / (1 - h2_fastlmm) g = BlockMatrix.from_entry_expr(mt_chr1.GT.n_alt_alleles()).to_numpy().T g_std = self._filter_and_standardize_cols(g) # full rank k = (g_std @ g_std.T) * (n / m) s, u = np.linalg.eigh(k) p = u.T model = LinearMixedModel(p @ y, p @ x, s) model.fit() assert np.isclose(model.h_sq, h2_fastlmm) h2_std_error = 0.13770773 # hard coded having checked against plot assert np.isclose(model.h_sq_standard_error, h2_std_error) h_sq_norm_lkhd = model.h_sq_normalized_lkhd()[1:-1] argmax = int(100 * h2_fastlmm) assert argmax <= np.argmax(h_sq_norm_lkhd) + 1 <= argmax + 1 assert np.isclose(np.sum(h_sq_norm_lkhd), 1.0) mt3_chr3_5var = mt_chr3.filter_rows(mt_chr3.locus.position < 2005) # first 5 a = BlockMatrix.from_entry_expr(mt3_chr3_5var.GT.n_alt_alleles()).to_numpy().T # FastLMM standardizes each variant to have mean 0 and variance 1. a = self._filter_and_standardize_cols(a) * np.sqrt(n) pa = p @ a model.fit(log_gamma=np.log(gamma_fastlmm)) res = model.fit_alternatives_numpy(pa).to_pandas() assert np.allclose(res['beta'], beta_fastlmm) assert np.allclose(res['p_value'], pval_hail) pa_t_path = utils.new_temp_file(suffix='bm') BlockMatrix.from_numpy(pa.T).write(pa_t_path, force_row_major=True) res = model.fit_alternatives(pa_t_path).to_pandas() assert np.allclose(res['beta'], beta_fastlmm) assert np.allclose(res['p_value'], pval_hail) # low rank l = g_std.T @ g_std sl, v = np.linalg.eigh(l) n_eigenvectors = int(np.sum(sl > 1e-10)) assert n_eigenvectors < n sl = sl[-n_eigenvectors:] v = v[:, -n_eigenvectors:] s = sl * (n / m) p = (g_std @ (v / np.sqrt(sl))).T model = LinearMixedModel(p @ y, p @ x, s, y, x) model.fit() assert np.isclose(model.h_sq, h2_fastlmm) assert np.isclose(model.h_sq_standard_error, h2_std_error) model.fit(log_gamma=np.log(gamma_fastlmm)) pa = p @ a res = model.fit_alternatives_numpy(pa, a).to_pandas() assert np.allclose(res['beta'], beta_fastlmm) assert np.allclose(res['p_value'], pval_hail) a_t_path = utils.new_temp_file(suffix='bm') BlockMatrix.from_numpy(a.T).write(a_t_path, force_row_major=True) pa_t_path = utils.new_temp_file(suffix='bm') BlockMatrix.from_numpy(pa.T).write(pa_t_path, force_row_major=True) res = model.fit_alternatives(pa_t_path, a_t_path).to_pandas() assert np.allclose(res['beta'], beta_fastlmm) assert np.allclose(res['p_value'], pval_hail) # testing chrom 3 for h2 h2_fastlmm = 0.36733240 g = BlockMatrix.from_entry_expr(mt_chr3.GT.n_alt_alleles()).to_numpy().T g_std = self._filter_and_standardize_cols(g) # full rank k = (g_std @ g_std.T) * (n / m) s, u = np.linalg.eigh(k) p = u.T model = LinearMixedModel(p @ y, p @ x, s) model.fit() assert np.isclose(model.h_sq, h2_fastlmm) h2_std_error = 0.17409641 # hard coded having checked against plot assert np.isclose(model.h_sq_standard_error, h2_std_error) h_sq_norm_lkhd = model.h_sq_normalized_lkhd()[1:-1] argmax = int(100 * h2_fastlmm) assert argmax <= np.argmax(h_sq_norm_lkhd) + 1 <= argmax + 1 assert np.isclose(np.sum(h_sq_norm_lkhd), 1.0) # low rank l = g_std.T @ g_std sl, v = np.linalg.eigh(l) n_eigenvectors = int(np.sum(sl > 1e-10)) assert n_eigenvectors < n sl = sl[-n_eigenvectors:] v = v[:, -n_eigenvectors:] s = sl * (n / m) p = (g_std @ (v / np.sqrt(sl))).T model = LinearMixedModel(p @ y, p @ x, s, y, x) model.fit() assert np.isclose(model.h_sq, h2_fastlmm) assert np.isclose(model.h_sq_standard_error, h2_std_error)
def _test_linear_mixed_model_full_rank(self): seed = 0 n_populations = 8 fst = n_populations * [.9] n_samples = 200 n_variants = 500 n_orig_markers = 500 n_culprits = 20 n_covariates = 3 sigma_sq = 1 tau_sq = 1 from numpy.random import RandomState prng = RandomState(seed) x = np.hstack((np.ones(shape=(n_samples, 1)), prng.normal(size=(n_samples, n_covariates - 1)))) mt = hl.balding_nichols_model(n_populations=n_populations, n_samples=n_samples, n_variants=n_variants, fst=fst, af_dist=hl.rand_unif(0.1, 0.9, seed=seed), seed=seed) pa_t_path = utils.new_temp_file(suffix='bm') a = BlockMatrix.from_entry_expr(mt.GT.n_alt_alleles()).T.to_numpy() g = a[:, -n_orig_markers:] g_std = self._filter_and_standardize_cols(g) n_markers = g_std.shape[1] k = (g_std @ g_std.T) * n_samples / n_markers beta = np.arange(n_covariates) beta_stars = np.array([1] * n_culprits) y = prng.multivariate_normal( np.hstack((a[:, 0:n_culprits], x)) @ np.hstack((beta_stars, beta)), sigma_sq * k + tau_sq * np.eye(n_samples)) s, u = np.linalg.eigh(k) p = u.T # build and fit model py = p @ y px = p @ x pa = p @ a model = LinearMixedModel(py, px, s) assert model.n == n_samples assert model.f == n_covariates assert model.r == n_samples assert (not model.low_rank) model.fit() # check effect sizes tend to be near 1 for first n_marker alternative models BlockMatrix.from_numpy(pa).T.write(pa_t_path, force_row_major=True) df_lmm = model.fit_alternatives(pa_t_path).to_pandas() assert 0.9 < np.mean(df_lmm['beta'][:n_culprits]) < 1.1 # compare NumPy and Hail LMM per alternative df_numpy = model.fit_alternatives_numpy(pa, a).to_pandas() assert np.min(df_numpy['chi_sq']) > 0 na_numpy = df_numpy.isna().any(axis=1) na_lmm = df_lmm.isna().any(axis=1) assert na_numpy.sum() <= 20 assert na_lmm.sum() <= 20 assert np.logical_xor(na_numpy, na_lmm).sum() <= 10 mask = ~(na_numpy | na_lmm) lmm_vs_numpy_p_value = np.sort(np.abs(df_lmm['p_value'][mask] - df_numpy['p_value'][mask])) assert lmm_vs_numpy_p_value[10] < 1e-12 # 10 least p-values differences assert lmm_vs_numpy_p_value[-1] < 1e-8 # all p-values
def ld_score(entry_expr, annotation_exprs, position_expr, window_size) -> Table: """Calculate LD scores. Example ------- >>> # Load genetic data into MatrixTable >>> mt = hl.import_plink(bed='data/ldsc.bed', ... bim='data/ldsc.bim', ... fam='data/ldsc.fam') >>> # Create locus-keyed Table with numeric variant annotations >>> ht = hl.import_table('data/ldsc.annot', ... types={'BP': hl.tint, ... 'binary': hl.tfloat, ... 'continuous': hl.tfloat}) >>> ht = ht.annotate(locus=hl.locus(ht.CHR, ht.BP)) >>> ht = ht.key_by('locus') >>> # Annotate MatrixTable with external annotations >>> mt = mt.annotate_rows(univariate_annotation=1, ... binary_annotation=ht[mt.locus].binary, ... continuous_annotation=ht[mt.locus].continuous) >>> # Annotate MatrixTable with alt allele count stats >>> mt = mt.annotate_rows(stats=hl.agg.stats(mt.GT.n_alt_alleles())) >>> # Create standardized genotype entry >>> mt = mt.annotate_entries(GT_std=hl.or_else( ... (mt.GT.n_alt_alleles() - mt.stats.mean)/mt.stats.stdev, 0.0)) >>> # Calculate LD scores using standardized genotypes >>> ht_scores = hl.experimental.ld_score(entry_expr=mt.GT_std, ... annotation_exprs=[ ... mt.univariate_annotation, ... mt.binary_annotation, ... mt.continuous_annotation], ... position_expr=mt.cm_position, ... window_size=1) Warning ------- :func:`.ld_score` will fail if ``entry_expr`` results in any missing values. The special float value ``nan`` is not considered a missing value. **Further reading** For more in-depth discussion of LD scores, see: - `LD Score regression distinguishes confounding from polygenicity in genome-wide association studies (Bulik-Sullivan et al, 2015) <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4495769/>`__ - `Partitioning heritability by functional annotation using genome-wide association summary statistics (Finucane et al, 2015) <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4626285/>`__ Parameters ---------- entry_expr : :class:`.NumericExpression` Expression for entries of genotype matrix (e.g. ``mt.GT.n_alt_alleles()``). annotation_exprs : :class:`.NumericExpression` or :obj:`list` of :class:`.NumericExpression` Annotation expression(s) to partition LD scores. position_expr : :class:`.NumericExpression` Expression for position of variant (e.g. ``mt.cm_position`` or ``mt.locus.position``). window_size : :obj:`int` or :obj:`float` Size of variant window used to calculate LD scores, in units of ``position``. Returns ------- :class:`.Table` Locus-keyed table with LD scores for each variant and annotation.""" assert window_size >= 0 mt = entry_expr._indices.source annotations = wrap_to_list(annotation_exprs) variant_key = [x for x in mt.row_key] ht_annotations = mt.select_rows(*annotations).rows() annotation_names = [x for x in ht_annotations.row if x not in variant_key] ht_annotations = hl.Table.union(*[(ht_annotations.annotate( annotation=hl.str(x), value=hl.float(ht_annotations[x])).select( 'annotation', 'value')) for x in annotation_names]) mt_annotations = ht_annotations.to_matrix_table(row_key=variant_key, col_key=['annotation']) cols = mt_annotations['annotation'].collect() col_idxs = {i: cols[i] for i in range(len(cols))} G = BlockMatrix.from_entry_expr(entry_expr) A = BlockMatrix.from_entry_expr(mt_annotations.value) n = G.n_cols R2 = ((G @ G.T) / n)**2 R2_adj = R2 - (1.0 - R2) / (n - 2.0) positions = [(x[0], float(x[1])) for x in hl.array([mt.locus.contig, hl.str(position_expr)]).collect()] n_positions = len(positions) starts = np.zeros(n_positions, dtype='int') stops = np.zeros(n_positions, dtype='int') contig = '0' for i, (c, p) in enumerate(positions): if c != contig: j = i k = i contig = c min_val = p - window_size max_val = p + window_size while j < n_positions and positions[j][1] < min_val: j += 1 starts[i] = j if k == n_positions: stops[i] = k continue while positions[k][0] == contig and positions[k][1] <= max_val: k += 1 if k == n_positions: break stops[i] = k R2_adj_sparse = R2_adj.sparsify_row_intervals([int(x) for x in starts], [int(x) for x in stops]) L2 = R2_adj_sparse @ A tmp_bm_path = new_temp_file() tmp_tsv_path = new_temp_file() L2.write(tmp_bm_path, force_row_major=True) BlockMatrix.export(tmp_bm_path, tmp_tsv_path) ht_scores = hl.import_table(tmp_tsv_path, no_header=True, impute=True) ht_scores = ht_scores.add_index() ht_scores = ht_scores.key_by('idx') ht_scores = ht_scores.rename( {'f{:}'.format(i): col_idxs[i] for i in range(len(cols))}) ht_variants = mt.rows() ht_variants = ht_variants.drop( *[x for x in ht_variants.row if x not in variant_key]) ht_variants = ht_variants.add_index() ht_variants = ht_variants.key_by('idx') ht_scores = ht_variants.join(ht_scores, how='inner') ht_scores = ht_scores.key_by('locus') ht_scores = ht_scores.drop('alleles', 'idx') return ht_scores
def test_linear_mixed_model_fastlmm(self): # FastLMM Test data is from all.bed, all.bim, all.fam, cov.txt, pheno_10_causals.txt: # https://github.com/MicrosoftGenomics/FaST-LMM/tree/master/tests/datasets/synth # # Data is filtered to chromosome 1,3 and samples 0-124,375-499 (2000 variants and 250 samples) # # Results are computed with single_snp (with LOCO) as in: # https://github.com/MicrosoftGenomics/FaST-LMM/blob/master/doc/ipynb/FaST-LMM.ipynb n, m = 250, 1000 # per chromosome x_table = hl.import_table(resource('fastlmmCov.txt'), no_header=True, impute=True).key_by('f1') y_table = hl.import_table(resource('fastlmmPheno.txt'), no_header=True, impute=True, delimiter=' ').key_by('f1') mt = hl.import_plink(bed=resource('fastlmmTest.bed'), bim=resource('fastlmmTest.bim'), fam=resource('fastlmmTest.fam'), reference_genome=None) mt = mt.annotate_cols(x=x_table[mt.col_key].f2) mt = mt.annotate_cols(y=y_table[mt.col_key].f2).cache() x = np.array([np.ones(n), mt.key_cols_by()['x'].collect()]).T y = np.array(mt.key_cols_by()['y'].collect()) mt_chr1 = mt.filter_rows(mt.locus.contig == '1') mt_chr3 = mt.filter_rows(mt.locus.contig == '3') # testing chrom 1 for h2, betas, p-values h2_fastlmm = 0.14276125 beta_fastlmm = [0.012202061, 0.037718282, -0.033572693, 0.29171541, -0.045644170] # FastLMM p-values do not agree to high precision because FastLMM regresses # out x from each SNP first and does an F(1, dof)-test on (beta / se)^2 # (t-test), whereas Hail does likelihood ratio test. # We verify below that Hail's p-values remain fixed going forward. # fastlmm = [0.84650294, 0.57865098, 0.59050998, 1.6649473e-06, 0.46892059] pval_hail = [0.84543084, 0.57596760, 0.58788517, 1.4057279e-06, 0.46578204] gamma_fastlmm = h2_fastlmm / (1 - h2_fastlmm) g = BlockMatrix.from_entry_expr(mt_chr1.GT.n_alt_alleles()).to_numpy().T g_std = self._filter_and_standardize_cols(g) # full rank k = (g_std @ g_std.T) * (n / m) s, u = np.linalg.eigh(k) p = u.T model = LinearMixedModel(p @ y, p @ x, s) model.fit() assert np.isclose(model.h_sq, h2_fastlmm) h2_std_error = 0.13770773 # hard coded having checked against plot assert np.isclose(model.h_sq_standard_error, h2_std_error) h_sq_norm_lkhd = model.h_sq_normalized_lkhd()[1:-1] argmax = int(100 * h2_fastlmm) assert argmax <= np.argmax(h_sq_norm_lkhd) + 1 <= argmax + 1 assert np.isclose(np.sum(h_sq_norm_lkhd), 1.0) mt3_chr3_5var = mt_chr3.filter_rows(mt_chr3.locus.position < 2005) # first 5 a = BlockMatrix.from_entry_expr(mt3_chr3_5var.GT.n_alt_alleles()).to_numpy().T # FastLMM standardizes each variant to have mean 0 and variance 1. a = self._filter_and_standardize_cols(a) * np.sqrt(n) pa = p @ a model.fit(log_gamma=np.log(gamma_fastlmm)) res = model.fit_alternatives_numpy(pa, return_pandas=True) assert np.allclose(res['beta'], beta_fastlmm) assert np.allclose(res['p_value'], pval_hail) pa_t_path = utils.new_temp_file(suffix='bm') BlockMatrix.from_numpy(pa.T).write(pa_t_path, force_row_major=True) res = model.fit_alternatives(pa_t_path).to_pandas() assert np.allclose(res['beta'], beta_fastlmm) assert np.allclose(res['p_value'], pval_hail) # low rank ld = g_std.T @ g_std sl, v = np.linalg.eigh(ld) n_eigenvectors = int(np.sum(sl > 1e-10)) assert n_eigenvectors < n sl = sl[-n_eigenvectors:] v = v[:, -n_eigenvectors:] s = sl * (n / m) p = (g_std @ (v / np.sqrt(sl))).T model = LinearMixedModel(p @ y, p @ x, s, y, x) model.fit() assert np.isclose(model.h_sq, h2_fastlmm) assert np.isclose(model.h_sq_standard_error, h2_std_error) model.fit(log_gamma=np.log(gamma_fastlmm)) pa = p @ a res = model.fit_alternatives_numpy(pa, a, return_pandas=True) assert np.allclose(res['beta'], beta_fastlmm) assert np.allclose(res['p_value'], pval_hail) a_t_path = utils.new_temp_file(suffix='bm') BlockMatrix.from_numpy(a.T).write(a_t_path, force_row_major=True) pa_t_path = utils.new_temp_file(suffix='bm') BlockMatrix.from_numpy(pa.T).write(pa_t_path, force_row_major=True) res = model.fit_alternatives(pa_t_path, a_t_path).to_pandas() assert np.allclose(res['beta'], beta_fastlmm) assert np.allclose(res['p_value'], pval_hail) # testing chrom 3 for h2 h2_fastlmm = 0.36733240 g = BlockMatrix.from_entry_expr(mt_chr3.GT.n_alt_alleles()).to_numpy().T g_std = self._filter_and_standardize_cols(g) # full rank k = (g_std @ g_std.T) * (n / m) s, u = np.linalg.eigh(k) p = u.T model = LinearMixedModel(p @ y, p @ x, s) model.fit() assert np.isclose(model.h_sq, h2_fastlmm) h2_std_error = 0.17409641 # hard coded having checked against plot assert np.isclose(model.h_sq_standard_error, h2_std_error) h_sq_norm_lkhd = model.h_sq_normalized_lkhd()[1:-1] argmax = int(100 * h2_fastlmm) assert argmax <= np.argmax(h_sq_norm_lkhd) + 1 <= argmax + 1 assert np.isclose(np.sum(h_sq_norm_lkhd), 1.0) # low rank l = g_std.T @ g_std sl, v = np.linalg.eigh(l) n_eigenvectors = int(np.sum(sl > 1e-10)) assert n_eigenvectors < n sl = sl[-n_eigenvectors:] v = v[:, -n_eigenvectors:] s = sl * (n / m) p = (g_std @ (v / np.sqrt(sl))).T model = LinearMixedModel(p @ y, p @ x, s, y, x) model.fit() assert np.isclose(model.h_sq, h2_fastlmm) assert np.isclose(model.h_sq_standard_error, h2_std_error)