def generate_final_rf_ht(
ht: hl.Table,
snp_cutoff: Union[int, float],
indel_cutoff: Union[int, float],
inbreeding_coeff_cutoff: float = INBREEDING_COEFF_HARD_CUTOFF,
determine_cutoff_from_bin: bool = False,
aggregated_bin_ht: Optional[hl.Table] = None,
bin_id: Optional[hl.expr.Int32Expression] = None,
) -> hl.Table:
"""
Prepares finalized RF model given an RF result table from `rf.apply_rf_model` and cutoffs for filtering.
If `determine_cutoff_from_bin` is True, `aggregated_bin_ht` must be supplied to determine the SNP and indel RF
probabilities to use as cutoffs from an aggregated quantile bin Table like one created by
`compute_grouped_binned_ht` in combination with `score_bin_agg`.
:param ht: RF result table from `rf.apply_rf_model` to prepare as the final RF Table
:param ac0_filter_expr: Expression that indicates if a variant should be filtered as allele count 0 (AC0)
:param ts_ac_filter_expr: Expression in `ht` that indicates if a variant is a transmitted singleton
:param mono_allelic_fiter_expr: Expression indicating if a variant is mono-allelic
:param snp_cutoff: RF probability or bin (if `determine_cutoff_from_bin` True) to use for SNP variant QC filter
:param indel_cutoff: RF probability or bin (if `determine_cutoff_from_bin` True) to use for indel variant QC filter
:param inbreeding_coeff_cutoff: InbreedingCoeff hard filter to use for variants
:param determine_cutoff_from_bin: If True RF probability will be determined using bin info in `aggregated_bin_ht`
:param aggregated_bin_ht: File with aggregate counts of variants based on quantile bins
:param bin_id: Name of bin to use in 'bin_id' column of `aggregated_bin_ht` to use to determine probability cutoff
:return: Finalized random forest Table annotated with variant filters
"""
# Determine SNP and indel RF cutoffs if given bin instead of RF probability
snp_cutoff_global = hl.struct(min_score=snp_cutoff)
indel_cutoff_global = hl.struct(min_score=indel_cutoff)
# Add filters to RF HT
filters = dict()
if ht.any(hl.is_missing(ht.rf_probability["TP"])):
raise ValueError("Missing RF probability!")
filters["RF"] = (
hl.is_snp(ht.alleles[0], ht.alleles[1])
& (ht.rf_probability["TP"] < snp_cutoff_global.min_score)) | (
~hl.is_snp(ht.alleles[0], ht.alleles[1])
& (ht.rf_probability["TP"] < indel_cutoff_global.min_score))
# Fix annotations for release
annotations_expr = {
"rf_positive_label": hl.or_else(ht.tp, False),
"rf_negative_label": ht.fail_hard_filters,
"rf_probability": ht.rf_probability["TP"],
}
ht = ht.transmute(filters=add_filters_expr(filters=filters),
**annotations_expr)
ht = ht.annotate_globals(rf_snv_cutoff=snp_cutoff_global,
rf_indel_cutoff=indel_cutoff_global)
return ht
def get_lowqual_expr(
alleles: hl.expr.ArrayExpression,
qual_approx_expr: Union[hl.expr.ArrayNumericExpression, hl.expr.NumericExpression],
snv_phred_threshold: int = 30,
snv_phred_het_prior: int = 30, # 1/1000
indel_phred_threshold: int = 30,
indel_phred_het_prior: int = 39, # 1/8,000
) -> Union[hl.expr.BooleanExpression, hl.expr.ArrayExpression]:
"""
Computes lowqual threshold expression for either split or unsplit alleles based on QUALapprox or AS_QUALapprox
.. note::
When running This lowqual annotation using QUALapprox, it differs from the GATK LowQual filter.
This is because GATK computes this annotation at the site level, which uses the least stringent prior for mixed sites.
When run using AS_QUALapprox, this implementation can thus be more stringent for certain alleles at mixed sites.
:param alleles: Array of alleles
:param qual_approx_expr: QUALapprox or AS_QUALapprox
:param snv_phred_threshold: Phred-scaled SNV "emission" threshold (similar to GATK emission threshold)
:param snv_phred_het_prior: Phred-scaled SNV heterozygosity prior (30 = 1/1000 bases, GATK default)
:param indel_phred_threshold: Phred-scaled indel "emission" threshold (similar to GATK emission threshold)
:param indel_phred_het_prior: Phred-scaled indel heterozygosity prior (30 = 1/1000 bases, GATK default)
:return: lowqual expression (BooleanExpression if `qual_approx_expr`is Numeric, Array[BooleanExpression] if `qual_approx_expr` is ArrayNumeric)
"""
min_snv_qual = snv_phred_threshold + snv_phred_het_prior
min_indel_qual = indel_phred_threshold + indel_phred_het_prior
min_mixed_qual = max(min_snv_qual, min_indel_qual)
if isinstance(qual_approx_expr, hl.expr.ArrayNumericExpression):
return hl.range(1, hl.len(alleles)).map(
lambda ai: hl.cond(
hl.is_snp(alleles[0], alleles[ai]),
qual_approx_expr[ai - 1] < min_snv_qual,
qual_approx_expr[ai - 1] < min_indel_qual,
)
)
else:
return (
hl.case()
.when(
hl.range(1, hl.len(alleles)).all(
lambda ai: hl.is_snp(alleles[0], alleles[ai])
),
qual_approx_expr < min_snv_qual,
)
.when(
hl.range(1, hl.len(alleles)).all(
lambda ai: hl.is_indel(alleles[0], alleles[ai])
),
qual_approx_expr < min_indel_qual,
)
.default(qual_approx_expr < min_mixed_qual)
)
def annotate_snp_mismatch(
t: Union[hl.MatrixTable, hl.Table], data_type: str,
rg: hl.genetics.ReferenceGenome) -> Union[hl.MatrixTable, hl.Table]:
"""
Annotates mismatches between reference allele and allele in reference fasta
Assumes input Table/MatrixTable has t.new_locus annotation
:param t: Table/MatrixTable of SNPs to be annotated
:param data_type: Data type (exomes or genomes for gnomAD; not used otherwise)
:param rg: Reference genome with fasta sequence loaded
:return: Table annotated with mismatches between reference allele and allele in fasta
"""
logger.info('Filtering to SNPs')
snp_expr = hl.is_snp(t.alleles[0], t.alleles[1])
t = t.filter(snp_expr) if isinstance(t,
hl.Table) else t.filter_rows(snp_expr)
mismatch_expr = {
'reference_mismatch':
hl.cond(t.new_locus.is_negative_strand,
(flip_base(t.alleles[0]) != hl.get_sequence(
t.locus.contig, t.locus.position, reference_genome=rg)),
(t.alleles[0] != hl.get_sequence(
t.locus.contig, t.locus.position, reference_genome=rg)))
}
logger.info(
'Checking if reference allele matches what is in reference fasta')
logger.info(
'For SNPs on the negative strand, make sure the reverse complement of the ref alleles matches what is in the ref fasta'
)
return t.annotate(**mismatch_expr) if isinstance(
t, hl.Table) else t.annotate_rows(**mismatch_expr)
def annotate_snp_mismatch(
t: Union[hl.MatrixTable, hl.Table],
rg: hl.genetics.ReferenceGenome) -> Union[hl.MatrixTable, hl.Table]:
"""
Annotates mismatches between reference allele and allele in reference fasta
Assumes input Table/MatrixTable has t.new_locus annotation
:param t: Table/MatrixTable of SNPs to be annotated
:param rg: Reference genome with fasta sequence loaded
:return: Table annotated with mismatches between reference allele and allele in fasta
"""
logger.info("Filtering to SNPs")
snp_expr = hl.is_snp(t.alleles[0], t.alleles[1])
t = t.filter(snp_expr) if isinstance(t,
hl.Table) else t.filter_rows(snp_expr)
mismatch_expr = {
"reference_mismatch":
hl.cond(
t.new_locus.is_negative_strand,
(hl.reverse_complement(t.alleles[0]) != hl.get_sequence(
t.locus.contig, t.locus.position, reference_genome=rg)),
(t.alleles[0] != hl.get_sequence(
t.locus.contig, t.locus.position, reference_genome=rg)),
)
}
logger.info(
"Checking if reference allele matches what is in reference fasta")
logger.info(
"For SNPs on the negative strand, make sure the reverse complement of the ref alleles matches what is in the ref fasta"
)
return (t.annotate(**mismatch_expr)
if isinstance(t, hl.Table) else t.annotate_rows(**mismatch_expr))
def generate_allele_data(ht: hl.Table) -> hl.Table:
"""
Returns bi-allelic sites HT with the following annotations:
- allele_data (nonsplit_alleles, has_star, variant_type, and n_alt_alleles)
:param Table ht: Full unsplit HT
:return: Table with allele data annotations
:rtype: Table
"""
ht = ht.select()
allele_data = hl.struct(nonsplit_alleles=ht.alleles,
has_star=hl.any(lambda a: a == "*", ht.alleles))
ht = ht.annotate(allele_data=allele_data.annotate(
**add_variant_type(ht.alleles)))
ht = hl.split_multi_hts(ht)
ht = ht.filter(hl.len(ht.alleles) > 1)
allele_type = (hl.case().when(
hl.is_snp(ht.alleles[0], ht.alleles[1]),
"snv").when(hl.is_insertion(ht.alleles[0], ht.alleles[1]),
"ins").when(hl.is_deletion(ht.alleles[0], ht.alleles[1]),
"del").default("complex"))
ht = ht.annotate(allele_data=ht.allele_data.annotate(
allele_type=allele_type,
was_mixed=ht.allele_data.variant_type == "mixed"))
return ht
def add_strand_flip_annotation(reference_ref, reference_alt, ds_a1, ds_a2):
""" Document me here :)
"""
is_strand_ambig = hl.is_strand_ambiguous(ds_a1, ds_a2)
ds_a1_flipped = flip_strand(ds_a1)
ds_a2_flipped = flip_strand(ds_a2)
is_snp = hl.is_snp(ds_a1, ds_a2)
null = hl.null(hl.tbool)
return (hl.case().when(
(ds_a1 == reference_alt) & (ds_a2 == reference_ref),
hl.cond(is_strand_ambig, [
hl.struct(swap=True, flip=True),
hl.struct(swap=False, flip=False)
], [hl.struct(swap=False, flip=False)])).when(
(ds_a1 == reference_ref) & (ds_a2 == reference_alt),
hl.cond(is_strand_ambig, [
hl.struct(swap=True, flip=False),
hl.struct(swap=False, flip=True)
], [hl.struct(swap=True, flip=False)])).when(
(ds_a1_flipped == reference_alt) &
(ds_a2_flipped == reference_ref) & is_snp,
[hl.struct(swap=False, flip=True)]).when(
(ds_a1_flipped == reference_ref) &
(ds_a2_flipped == reference_alt) & is_snp,
[hl.struct(swap=True, flip=True)]).default(
hl.empty_array(hl.tstruct(swap=hl.tbool,
flip=hl.tbool))))
def generate_allele_data(mt: hl.MatrixTable) -> hl.Table:
"""
Writes bi-allelic sites MT with the following annotations:
- allele_data (nonsplit_alleles, has_star, variant_type, and n_alt_alleles)
:param MatrixTable mt: Full unsplit MT
:return: Table with allele data annotations
:rtype: Table
"""
ht = mt.rows().select()
allele_data = hl.struct(nonsplit_alleles=ht.alleles,
has_star=hl.any(lambda a: a == '*', ht.alleles))
ht = ht.annotate(allele_data=allele_data.annotate(
**add_variant_type(ht.alleles)))
ht = hl.split_multi_hts(ht)
allele_type = (hl.case().when(
hl.is_snp(ht.alleles[0], ht.alleles[1]),
'snv').when(hl.is_insertion(ht.alleles[0], ht.alleles[1]),
'ins').when(hl.is_deletion(ht.alleles[0], ht.alleles[1]),
'del').default('complex'))
ht = ht.annotate(allele_data=ht.allele_data.annotate(
allele_type=allele_type,
was_mixed=ht.allele_data.variant_type == 'mixed'))
return ht
def concordance_frequency(full_vcf, concordance_table, output):
full_variant_qc = full_vcf.rows()
concordance_qc = full_variant_qc.annotate(
concordance=concordance_table[full_variant_qc.key])
freqs = list(np.linspace(0.5, 0,
num=91)) ## note, this will need to be updated
concordance_stats = concordance_qc.group_by(
freq=hl.array(freqs).find(
lambda x: concordance_qc.variant_qc.AF[1] >= x),
snp=hl.is_snp(
concordance_qc.alleles[0], concordance_qc.alleles[1])).aggregate(
n_variants=hl.agg.count(),
unique_variants=hl.agg.array_agg(
lambda row: hl.agg.array_agg(
lambda element: hl.agg.count_where(element > 0), row),
concordance_qc.concordance.concordance),
geno_concordance=hl.agg.array_agg(
lambda row: hl.agg.array_agg(
lambda element: hl.agg.sum(element), row),
concordance_qc.concordance.concordance))
concordance_stats = concordance_stats.annotate(
total_concordant=concordance_stats.geno_concordance[3][3] +
concordance_stats.geno_concordance[4][4],
total_discordant=concordance_stats.geno_concordance[2][3] +
concordance_stats.geno_concordance[2][4] +
concordance_stats.geno_concordance[3][2] +
concordance_stats.geno_concordance[3][4] +
concordance_stats.geno_concordance[4][2] +
concordance_stats.geno_concordance[4][3])
concordance_stats = concordance_stats.annotate(
non_ref_concordance=concordance_stats.total_concordant /
(concordance_stats.total_concordant +
concordance_stats.total_discordant))
concordance_stats.export(output + 'variants.tsv')
def low_qual_expr(
ref: hl.expr.StringExpression,
alt: hl.expr.StringExpression,
qual_approx: hl.expr.NumericExpression,
) -> hl.expr.BooleanExpression:
return hl.cond(
hl.is_snp(ref, alt),
qual_approx < snv_phred_threshold + snv_phred_het_prior,
qual_approx < indel_phred_threshold + indel_phred_het_prior,
)
def split_mt_to_snps(mt: hl.MatrixTable) -> hl.MatrixTable:
'''
:param mt: hail matrixtable of all samples with both indels and SNVs
:return: matrixtable with only the SNPs from all the samples
'''
mt_snps = hl.filter_alleles(
mt, lambda allele, _: hl.is_snp(mt.alleles[0], allele))
return mt_snps
def create_truth_sample_ht(
mt: hl.MatrixTable, truth_mt: hl.MatrixTable, high_confidence_intervals_ht: hl.Table
) -> hl.Table:
"""
Computes a table comparing a truth sample in callset vs the truth.
:param mt: MT of truth sample from callset to be compared to truth
:param truth_mt: MT of truth sample
:param high_confidence_intervals_ht: High confidence interval HT
:return: Table containing both the callset truth sample and the truth data
"""
def split_filter_and_flatten_ht(
truth_mt: hl.MatrixTable, high_confidence_intervals_ht: hl.Table
) -> hl.Table:
"""
Splits a truth sample MT and filter it to the given high confidence intervals.
Then "flatten" it as a HT by annotating GT in a row field.
:param truth_mt: Truth sample MT
:param high_confidence_intervals_ht: High confidence intervals
:return: Truth sample table with GT as a row annotation
"""
assert truth_mt.count_cols() == 1
if not "was_split" in truth_mt.row:
truth_mt = hl.split_multi_hts(truth_mt)
truth_mt = truth_mt.filter_rows(
hl.is_defined(high_confidence_intervals_ht[truth_mt.locus])
)
rename_entries = {"GT": "_GT"}
if "adj" in truth_mt.entry:
rename_entries.update({"adj": "_adj"})
truth_mt = truth_mt.rename(rename_entries)
return truth_mt.annotate_rows(
**{x: hl.agg.take(truth_mt[f"_{x}"], 1)[0] for x in rename_entries}
).rows()
# Load truth sample MT,
# restrict it to high confidence intervals
# and flatten it to a HT by annotating GT in a row annotation
truth_ht = split_filter_and_flatten_ht(truth_mt, high_confidence_intervals_ht)
truth_ht = truth_ht.rename({f: f"truth_{f}" for f in truth_ht.row_value})
# Similarly load, filter and flatten callset truth sample MT
ht = split_filter_and_flatten_ht(mt, high_confidence_intervals_ht)
# Outer join of truth and callset truth and annotate the score and global bin
ht = truth_ht.join(ht, how="outer")
ht = ht.annotate(snv=hl.is_snp(ht.alleles[0], ht.alleles[1]))
return ht
def main(args):
mt = get_gnomad_v3_mt(key_by_locus_and_alleles=True)
mt = mt.annotate_entries(
gvcf_info=mt.gvcf_info.drop('ClippingRankSum', 'ReadPosRankSum'))
mt = mt.annotate_rows(
n_unsplit_alleles=hl.len(mt.alleles),
mixed_site=(hl.len(mt.alleles) > 2)
& hl.any(lambda a: hl.is_indel(mt.alleles[0], a), mt.alleles[1:])
& hl.any(lambda a: hl.is_snp(mt.alleles[0], a), mt.alleles[1:]))
mt = hl.experimental.sparse_split_multi(mt, filter_changed_loci=True)
mt.write(args.split_mt_location, overwrite=args.overwrite)
def add_rank(
ht: hl.Table,
score_expr: hl.expr.NumericExpression,
subrank_expr: Optional[Dict[str, hl.expr.BooleanExpression]] = None,
) -> hl.Table:
"""
Adds rank based on the `score_expr`. Rank is added for snvs and indels separately.
If one or more `subrank_expr` are provided, then subrank is added based on all sites for which the boolean expression is true.
In addition, variant counts (snv, indel separately) is added as a global (`rank_variant_counts`).
:param ht: input Hail Table containing variants (with QC annotations) to be ranked
:param score_expr: the Table annotation by which ranking should be scored
:param subrank_expr: Any subranking to be added in the form name_of_subrank: subrank_filtering_expr
:return: Table with rankings added
"""
key = ht.key
if subrank_expr is None:
subrank_expr = {}
temp_expr = {"_score": score_expr}
temp_expr.update({f"_{name}": expr for name, expr in subrank_expr.items()})
rank_ht = ht.select(
**temp_expr, is_snv=hl.is_snp(ht.alleles[0], ht.alleles[1]))
rank_ht = rank_ht.key_by("_score").persist()
scan_expr = {
"rank": hl.cond(
rank_ht.is_snv,
hl.scan.count_where(rank_ht.is_snv),
hl.scan.count_where(~rank_ht.is_snv),
)
}
scan_expr.update(
{
name: hl.or_missing(
rank_ht[f"_{name}"],
hl.cond(
rank_ht.is_snv,
hl.scan.count_where(rank_ht.is_snv & rank_ht[f"_{name}"]),
hl.scan.count_where(~rank_ht.is_snv & rank_ht[f"_{name}"]),
),
)
for name in subrank_expr
}
)
rank_ht = rank_ht.annotate(**scan_expr)
rank_ht = rank_ht.key_by(*key).persist()
rank_ht = rank_ht.select(*scan_expr.keys())
ht = ht.annotate(**rank_ht[key])
return ht
def main(args):
print("main")
run_hash = "91b132aa"
ht = hl.read_table(
f'{temp_dir}/ddd-elgh-ukbb/variant_qc/models/{run_hash}/{run_hash}_rf_result_ranked_denovo_ddd_comp.ht'
)
mt = hl.read_matrix_table(
f'{temp_dir}/ddd-elgh-ukbb/Sanger_cohorts_chr1-7and20_split_sampleqc_filtered.mt'
)
mt = mt.annotate_rows(Variant_Type=hl.cond(
(hl.is_snp(mt.alleles[0], mt.alleles[1])), "SNP",
hl.cond(
hl.is_insertion(mt.alleles[0], mt.alleles[1]), "INDEL",
hl.cond(hl.is_deletion(mt.alleles[0], mt.alleles[1]), "INDEL",
"Other"))))
mt = mt.annotate_rows(info=mt.info.annotate(
rf_probability=ht[mt.row_key].rf_probability['TP']))
mt = mt.annotate_rows(info=mt.info.annotate(score=ht[mt.row_key].score))
filter_column_annotation = (
hl.case().when(
((mt.Variant_Type == "SNP") & (mt.info.rf_probability <= 0.90)),
"PASS").when(((mt.Variant_Type == "INDEL") &
(mt.info.rf_probability <= 0.80)),
"PASS").default(".") # remove everything else
)
# mt_annotated = mt.annotate_rows(mt.filters=filter_column_annotation)
mt1 = mt.annotate_rows(filtercol=((filter_column_annotation)))
mt_fail = mt1.filter_rows(mt1.filtercol == ".")
print(mt_fail.count())
mt2 = mt1.annotate_rows(filters=mt1.filters.add(mt1.filtercol))
mt_fail2 = mt2.filter_rows(mt2.filters.contains("."))
mt_pass = mt2.filter_rows(mt2.filters.contains("PASS"))
print(mt_fail2.count())
print(mt_pass.count())
mt2 = mt2.checkpoint(
f'{tmp_dir}/Sanger_cohorts_chr1-7and20_after_RF_final.mt',
overwrite=True)
hl.export_vcf(
mt2,
f'{tmp_dir}/Sanger_cohorts_chr1-7and20_after_RF_final.vcf.bgz',
parallel='separate_header')
def add_variant_type(alt_alleles: hl.expr.ArrayExpression) -> hl.expr.StructExpression:
"""
Get Struct of variant_type and n_alt_alleles from ArrayExpression of Strings (all alleles)
"""
ref = alt_alleles[0]
alts = alt_alleles[1:]
non_star_alleles = hl.filter(lambda a: a != '*', alts)
return hl.struct(variant_type=hl.cond(
hl.all(lambda a: hl.is_snp(ref, a), non_star_alleles),
hl.cond(hl.len(non_star_alleles) > 1, "multi-snv", "snv"),
hl.cond(
hl.all(lambda a: hl.is_indel(ref, a), non_star_alleles),
hl.cond(hl.len(non_star_alleles) > 1, "multi-indel", "indel"),
"mixed")
), n_alt_alleles=hl.len(non_star_alleles))
def get_gnomad_v3_mt(
split=False,
key_by_locus_and_alleles: bool = False,
remove_hard_filtered_samples: bool = True,
release_only: bool = False,
samples_meta: bool = False,
) -> hl.MatrixTable:
"""
Wrapper function to get gnomAD data with desired filtering and metadata annotations
:param split: Perform split on MT - Note: this will perform a split on the MT rather than grab an already split MT
:param key_by_locus_and_alleles: Whether to key the MatrixTable by locus and alleles (only needed for v3)
:param remove_hard_filtered_samples: Whether to remove samples that failed hard filters (only relevant after sample QC)
:param release_only: Whether to filter the MT to only samples available for release (can only be used if metadata is present)
:param samples_meta: Whether to add metadata to MT in 'meta' column
:return: gnomAD v3 dataset with chosen annotations and filters
"""
mt = gnomad_v3_genotypes.mt()
if key_by_locus_and_alleles:
mt = hl.MatrixTable(
hl.ir.MatrixKeyRowsBy(
mt._mir, ["locus", "alleles"], is_sorted=True
) # Prevents hail from running sort on genotype MT which is already sorted by a unique locus
)
if remove_hard_filtered_samples:
mt = mt.filter_cols(
hl.is_missing(hard_filtered_samples.ht()[mt.col_key]))
if samples_meta:
mt = mt.annotate_cols(meta=meta.ht()[mt.col_key])
if release_only:
mt = mt.filter_cols(mt.meta.release)
elif release_only:
mt = mt.filter_cols(meta.ht()[mt.col_key].release)
if split:
mt = mt.annotate_rows(
n_unsplit_alleles=hl.len(mt.alleles),
mixed_site=(hl.len(mt.alleles) > 2)
& hl.any(lambda a: hl.is_indel(mt.alleles[0], a), mt.alleles[1:])
& hl.any(lambda a: hl.is_snp(mt.alleles[0], a), mt.alleles[1:]),
)
mt = hl.experimental.sparse_split_multi(mt, filter_changed_loci=True)
return mt
def compute_grouped_binned_ht(
bin_ht: hl.Table,
checkpoint_path: Optional[str] = None,
) -> hl.GroupedTable:
"""
Group a Table that has been annotated with bins (`compute_ranked_bin` or `create_binned_ht`).
The table will be grouped by bin_id (bin, biallelic, etc.), contig, snv, bi_allelic and singleton.
.. note::
If performing an aggregation following this grouping (such as `score_bin_agg`) then the aggregation
function will need to use `ht._parent` to get the origin Table from the GroupedTable for the aggregation
:param bin_ht: Input Table with a `bin_id` annotation
:param checkpoint_path: If provided an intermediate checkpoint table is created with all required annotations before shuffling.
:return: Table grouped by bins(s)
"""
# Explode the rank table by bin_id
bin_ht = bin_ht.annotate(
bin_groups=hl.array(
[
hl.Struct(bin_id=bin_name, bin=bin_ht[bin_name])
for bin_name in bin_ht.bin_group_variant_counts
]
)
)
bin_ht = bin_ht.explode(bin_ht.bin_groups)
bin_ht = bin_ht.transmute(
bin_id=bin_ht.bin_groups.bin_id, bin=bin_ht.bin_groups.bin
)
bin_ht = bin_ht.filter(hl.is_defined(bin_ht.bin))
if checkpoint_path is not None:
bin_ht.checkpoint(checkpoint_path, overwrite=True)
else:
bin_ht = bin_ht.persist()
# Group by bin_id, bin and additional stratification desired and compute QC metrics per bin
return bin_ht.group_by(
bin_id=bin_ht.bin_id,
contig=bin_ht.locus.contig,
snv=hl.is_snp(bin_ht.alleles[0], bin_ht.alleles[1]),
bi_allelic=~bin_ht.was_split,
singleton=bin_ht.singleton,
release_adj=bin_ht.ac > 0,
bin=bin_ht.bin,
)._set_buffer_size(20000)
def get_doubleton_sites(
vds_path: str = VDS_PATH,
temp_path: str = TEMP_PATH,
tranche_data: Tuple[str, int] = TRANCHE_DATA,
sparse_entries: List[str] = SPARSE_ENTRIES,
) -> hl.Table:
"""
Filter UKB VDS to bi-allelic, autosomal sites in interval QC pass regions with an adj allele count of two and no homozygotes.
:param vds_path: Path to UKB 455k VDS. Default is VDS_PATH.
:param temp_path: Path to bucket to store Table and other temporary data. Default is TEMP_PATH.
:param tranche_data: UKB tranche data (data source and data freeze number). Default is TRANCHE_DATA.
:param sparse_entries: List of fields to select from VDS. Default is SPARSE_ENTRIES.
:return: Table of high quality sites with doubletons.
"""
logger.info("Reading in VDS and filtering to bi-allelic SNPs...")
mt = hl.vds.read_vds(vds_path).variant_data
# Drop unnecessary annotations
mt = mt.select_rows().select_entries(*sparse_entries)
mt = mt.filter_rows(
bi_allelic_expr(mt) & hl.is_snp(mt.alleles[0], mt.alleles[1]))
logger.info("Filter to autosomes and splitting multiallelics...")
mt = mt.filter_rows(mt.locus.in_autosome())
# NOTE: UKB dataset does not have errors with changed loci
# (`filter_changed_loci = False` will not throw errors here)
mt = hl.experimental.sparse_split_multi(mt)
logger.info("Removing AS_lowqual sites...")
info_ht = hl.read_table(info_ht_path(*tranche_data, split=True))
mt = mt.filter_rows(~info_ht[mt.row_key].AS_lowqual)
logger.info("Filtering to interval QC pass regions...")
interval_ht = hl.read_table(interval_qc_path(*tranche_data, "autosomes"))
mt = mt.filter_rows(hl.is_defined(interval_ht[mt.locus]))
logger.info("Filtering to adj and calculating allele count...")
mt = filter_to_adj(mt)
mt = mt.annotate_rows(call_stats=hl.agg.call_stats(mt.GT, mt.alleles))
# Get AC at allele index 1 (call_stats includes a count for each allele, including reference)
mt = mt.transmute_rows(ac=mt.call_stats.AC[1],
n_hom=mt.call_stats.homozygote_count[1])
logger.info("Filtering to an allele count of two and returning...")
ht = mt.rows()
ht = ht.filter((ht.ac == 2) & (ht.n_hom == 0))
ht = ht.checkpoint(f"{temp_path}/high_quality_sites.ht", overwrite=True)
return ht
def create_aggregated_bin_ht(model_id: str) -> hl.Table:
"""
Aggregates variants into bins, grouped by `bin_id` (rank, bi-allelic, etc.), contig, and `snv`, `bi_allelic`,
and `singleton` status, using previously annotated bin information.
For each bin, aggregates statistics needed for evaluation plots.
:param str model_id: Which variant QC model (RF or VQSR model ID) to group
:return: Table of aggregate statistics by bin
"""
ht = get_score_bins(model_id, aggregated=False).ht()
# Count variants for ranking
count_expr = {
x: hl.agg.filter(
hl.is_defined(ht[x]),
hl.agg.counter(
hl.cond(hl.is_snp(ht.alleles[0], ht.alleles[1]), "snv",
"indel")),
)
for x in ht.row if x.endswith("bin")
}
bin_variant_counts = ht.aggregate(hl.struct(**count_expr))
logger.info(
f"Found the following variant counts:\n {pformat(bin_variant_counts)}")
ht = ht.annotate_globals(bin_variant_counts=bin_variant_counts)
# Load ClinVar pathogenic data
clinvar_pathogenic_ht = filter_to_clinvar_pathogenic(clinvar.ht())
ht = ht.annotate(clinvar_path=hl.is_defined(clinvar_pathogenic_ht[ht.key]))
trio_stats_ht = fam_stats.ht()
logger.info(f"Creating grouped bin table...")
grouped_binned_ht = compute_grouped_binned_ht(
ht,
checkpoint_path=get_checkpoint_path(f"grouped_bin_{model_id}"),
)
logger.info(f"Aggregating grouped bin table...")
parent_ht = grouped_binned_ht._parent
agg_ht = grouped_binned_ht.aggregate(
n_clinvar_path=hl.agg.count_where(parent_ht.clinvar_path),
**score_bin_agg(grouped_binned_ht, fam_stats_ht=trio_stats_ht),
)
return agg_ht
def write_ldsc_hm3_snplist(info_threshold=0.9,
maf_threshold=0.01,
overwrite=False):
# Filter variants
ht = hl.read_table(get_variant_results_qc_path())
# in autosomes
ht = ht.filter(ht.locus.in_autosome())
# no MHC
ht = ht.filter(
~hl.parse_locus_interval('6:28477797-33448354').contains(ht.locus))
# info > 0.9
ht = ht.filter(ht.info > info_threshold)
# SNP only
ht = ht.filter(hl.is_snp(ht.alleles[0], ht.alleles[1]))
# no multi-allelic sites
loc_count = ht.group_by(ht.locus).aggregate(nloc=hl.agg.count())
loc_count = loc_count.filter(loc_count.nloc > 1)
multi_sites = loc_count.aggregate(hl.agg.collect_as_set(loc_count.locus),
_localize=False)
ht = ht.filter(~multi_sites.contains(ht.locus))
# in HM3
hm3_snps = hl.read_table(
'gs://ukbb-ldsc-dev/ukb_hm3_snplist/hm3.r3.b37.auto_bi_af.ht')
hm3_snps = hm3_snps.select()
ht = ht.join(hm3_snps, 'right')
# no strand ambiguity
ht = ht.filter(~hl.is_strand_ambiguous(ht.alleles[0], ht.alleles[1]))
ht = checkpoint_tmp(ht)
def get_maf(af):
return 0.5 - hl.abs(0.5 - af)
# MAF > 1% in UKB & gnomad genome/exome (if defined) for each population
for pop in POPS:
snplist = ht.filter(
hl.rbind(
ht.freq[ht.freq.index(lambda x: x.pop == pop)], lambda y:
(get_maf(y.af) > maf_threshold) &
(hl.is_missing(y.gnomad_genomes_af) |
(get_maf(y.gnomad_genomes_af) > maf_threshold)) &
(hl.is_missing(y.gnomad_exomes_af) |
(get_maf(y.gnomad_exomes_af) > maf_threshold))))
snplist = snplist.select('rsid')
snplist.write(get_hm3_snplist_path(pop), overwrite=overwrite)
def generate_split_alleles(mt: hl.MatrixTable) -> hl.Table:
allele_data = hl.struct(nonsplit_alleles=mt.alleles,
has_star=hl.any(lambda a: a == '*', mt.alleles))
mt = mt.annotate_rows(allele_data=allele_data.annotate(
**add_variant_type(mt.alleles)))
mt = hl.split_multi_hts(mt, left_aligned=True)
allele_type = (hl.case().when(
hl.is_snp(mt.alleles[0], mt.alleles[1]),
'snv').when(hl.is_insertion(mt.alleles[0], mt.alleles[1]),
'ins').when(hl.is_deletion(mt.alleles[0], mt.alleles[1]),
'del').default('complex'))
mt = mt.annotate_rows(allele_data=mt.allele_data.annotate(
allele_type=allele_type,
was_mixed=mt.allele_data.variant_type == 'mixed'))
return mt
def main(args):
mt = hl.read_matrix_table(args.matrixtable)
# From gnomad apply hard filters:
mt = mt.filter_rows((hl.len(mt.alleles) == 2)
& hl.is_snp(mt.alleles[0], mt.alleles[1])
& (hl.agg.mean(mt.GT.n_alt_alleles()) / 2 > 0.001)
& (hl.agg.fraction(hl.is_defined(mt.GT)) > 0.99))
mt.annotate_cols(callrate=hl.agg.fraction(hl.is_defined(mt.GT))).write(
f"{args.output_dir}/mt_hard_filters_annotated.mt", overwrite=True)
print("Sex imputation:")
mt_sex = annotate_sex(mt,
f"{args.output_dir}/sex_annotated",
male_threshold=0.6)
mt_sex.write(f"{args.output_dir}/mt_sex_annotated.mt", overwrite=True)
qc_ht = mt_sex.cols()
qc_ht = qc_ht.annotate(
ambiguous_sex=((qc_ht.f_stat >= 0.5) &
(hl.is_defined(qc_ht.normalized_y_coverage) &
(qc_ht.normalized_y_coverage <= 0.1))) |
(hl.is_missing(qc_ht.f_stat)) |
((qc_ht.f_stat >= 0.4) & (qc_ht.f_stat <= 0.6) &
(hl.is_defined(qc_ht.normalized_y_coverage) &
(qc_ht.normalized_y_coverage > 0.1))),
sex_aneuploidy=(qc_ht.f_stat < 0.4)
& hl.is_defined(qc_ht.normalized_y_coverage) &
(qc_ht.normalized_y_coverage > 0.1))
print("Annotating samples failing hard filters:")
logger.info("Annotating samples failing hard filters...")
sex_expr = (hl.case().when(qc_ht.ambiguous_sex, "ambiguous_sex").when(
qc_ht.sex_aneuploidy, "sex_aneuploidy").when(qc_ht.is_female,
"female").default("male"))
qc_ht = qc_ht.annotate(sex=sex_expr,
data_type='exomes').key_by('data_type', 's')
qc_ht.write(f"{args.output_dir}/mt_ambiguous_sex_samples.ht",
overwrite=True)
def create_grouped_bin_ht(model_id: str, overwrite: bool = False) -> None:
"""
Creates binned data from a quantile bin annotated Table grouped by bin_id (rank, bi-allelic, etc.), contig, snv,
bi_allelic and singleton containing the information needed for evaluation plots.
:param str model_id: Which data/run hash is being created
:param bool overwrite: Should output files be overwritten if present
:return: None
:rtype: None
"""
ht = get_score_quantile_bins(model_id, aggregated=False).ht()
# Count variants for ranking
count_expr = {
x: hl.agg.filter(
hl.is_defined(ht[x]),
hl.agg.counter(
hl.cond(hl.is_snp(ht.alleles[0], ht.alleles[1]), "snv",
"indel")),
)
for x in ht.row if x.endswith("bin")
}
bin_variant_counts = ht.aggregate(hl.struct(**count_expr))
logger.info(
f"Found the following variant counts:\n {pformat(bin_variant_counts)}")
ht = ht.annotate_globals(bin_variant_counts=bin_variant_counts)
trio_stats_ht = fam_stats.ht()
logger.info(f"Creating grouped bin table...")
grouped_binned_ht = compute_grouped_binned_ht(
ht,
checkpoint_path=get_checkpoint_path(f"grouped_bin_{model_id}"),
)
logger.info(f"Aggregating grouped bin table...")
agg_ht = grouped_binned_ht.aggregate(
**score_bin_agg(grouped_binned_ht, fam_stats_ht=trio_stats_ht))
agg_ht.write(
get_score_quantile_bins(model_id, aggregated=True).path,
overwrite=overwrite,
)
def compute_kinship_ht(mt, genome_version="GRCh38"):
mt = filter_to_biallelics(mt)
mt = filter_to_autosomes(mt)
mt = mt.filter_rows(hl.is_snp(mt.alleles[0], mt.alleles[1]))
mt = hl.variant_qc(mt)
mt = mt.filter_rows(mt.variant_qc.call_rate > 0.99)
#mt = mt.filter_rows(mt.info.AF > 0.001) # leaves 100% of variants
mt = ld_prune(mt, genome_version=genome_version)
ibd_results_ht = hl.identity_by_descent(mt,
maf=mt.info.AF,
min=0.10,
max=1.0)
ibd_results_ht = ibd_results_ht.annotate(
ibd0=ibd_results_ht.ibd.Z0,
ibd1=ibd_results_ht.ibd.Z1,
ibd2=ibd_results_ht.ibd.Z2,
pi_hat=ibd_results_ht.ibd.PI_HAT).drop("ibs0", "ibs1", "ibs2", "ibd")
kin_ht = ibd_results_ht
# filter to anything above the relationship of a grandparent
first_degree_pi_hat = .40
grandparent_pi_hat = .20
grandparent_ibd1 = 0.25
grandparent_ibd2 = 0.15
kin_ht = kin_ht.key_by("i", "j")
kin_ht = kin_ht.filter((kin_ht.pi_hat > first_degree_pi_hat) | (
(kin_ht.pi_hat > grandparent_pi_hat) & (kin_ht.ibd1 > grandparent_ibd1)
& (kin_ht.ibd2 < grandparent_ibd2)))
kin_ht = kin_ht.annotate(relation=hl.sorted([kin_ht.i, kin_ht.j
])) #better variable name
return kin_ht
PHENOTYPES_TABLE = 'gs://dalio_bipolar_w1_w2_hail_02/data/samples/phenotypes.ht'
ANNOTATION_TABLE = 'gs://dalio_bipolar_w1_w2_hail_02/data/annotations/gene.ht'
URV_NOT_IN_GNOMAD_FILE = 'gs://dalio_bipolar_w1_w2_hail_02/data/samples/20_URVs_not_in_gnomAD.tsv'
NOT_IN_GNOMAD_FILE = 'gs://dalio_bipolar_w1_w2_hail_02/data/samples/20_not_in_gnomAD.tsv'
mt = hl.read_matrix_table(MT)
sample_annotations = hl.read_table(PHENOTYPES_TABLE)
constraint_annotations = hl.read_table(ANNOTATION_TABLE)
mt = mt.annotate_cols(phenotype = sample_annotations[mt.s])
mt = mt.annotate_rows(constraint = constraint_annotations[mt.row_key])
mt = mt.filter_rows(~mt.constraint.inGnomAD_nonpsych)
mt = mt.annotate_cols(n_SNP = hl.agg.count_where(mt.GT.is_non_ref() & hl.is_snp(mt.alleles[0], mt.alleles[1])),
n_indel = hl.agg.count_where(mt.GT.is_non_ref() & hl.is_indel(mt.alleles[0], mt.alleles[1])),
n_coding_SNP = hl.agg.count_where(mt.GT.is_non_ref() & hl.is_snp(mt.alleles[0], mt.alleles[1]) & (mt.constraint.consequence_category != "non_coding")),
n_coding_indel = hl.agg.count_where(mt.GT.is_non_ref() & hl.is_indel(mt.alleles[0], mt.alleles[1]) & (mt.constraint.consequence_category != "non_coding")),
n_PTV = hl.agg.count_where(mt.GT.is_non_ref() & (mt.constraint.consequence_category == "ptv")),
n_damaging_missense = hl.agg.count_where(mt.GT.is_non_ref() & (mt.constraint.consequence_category == "damaging_missense")),
n_other_missense = hl.agg.count_where(mt.GT.is_non_ref() & (mt.constraint.consequence_category == "other_missense")),
n_synonymous = hl.agg.count_where(mt.GT.is_non_ref() & (mt.constraint.consequence_category == "synonymous")),
n_non_coding = hl.agg.count_where(mt.GT.is_non_ref() & (mt.constraint.consequence_category == "non_coding")))
mt.cols().flatten().export(NOT_IN_GNOMAD_FILE)
mt = mt.annotate_rows(is_singleton = hl.agg.sum(mt.GT.n_alt_alleles()) == 1)
mt = mt.filter_rows(mt.is_singleton)
mt = mt.annotate_cols(n_URV_SNP = hl.agg.count_where(mt.GT.is_non_ref() & hl.is_snp(mt.alleles[0], mt.alleles[1])),
def matrix_table_rows_is_transition(mt_path):
ht = hl.read_matrix_table(mt_path).rows().key_by()
ht.select(is_snp=hl.is_snp(ht.alleles[0], ht.alleles[1]))._force_count()
import hail.expr.aggregators as agg
hl.init()
#read mt file
mt = hl.read_matrix_table(
"gs://1k_genome/1000-genomes/VDS-of-all/ALL.chr.integrated_phase1_v3.20101123.snps_indels_svs.genotypes.mt"
)
#print(mt.count()) (39706715, 1092)
#filter MAF
mt = hl.variant_qc(mt)
mt = mt.filter_rows(mt.variant_qc.AF[1] > 0.01)
#print(mt.count()) (13404583, 1092)
#filter only SNPs
mt = mt.filter_rows(hl.is_snp(mt.alleles[0], mt.alleles[1]))
#print(mt.count()) (12194564, 1092)
#annotate MT file
table = (hl.import_table('gs://ines-work/KG-annotation-with-sexencoder.csv',
delimiter=',',
missing='',
quote='"',
types={
'Gender_Classification': hl.tfloat64
}).key_by('Sample'))
mt = mt.annotate_cols(**table[mt.s])
#print(mt.aggregate_cols(agg.counter(mt.Gender_Classification))) {'0.0': 567, '1.0': 525}
#pca
def annotate_sex(
mtds: Union[hl.MatrixTable, hl.vds.VariantDataset],
is_sparse: bool = True,
excluded_intervals: Optional[hl.Table] = None,
included_intervals: Optional[hl.Table] = None,
normalization_contig: str = "chr20",
reference_genome: str = "GRCh38",
sites_ht: Optional[hl.Table] = None,
aaf_expr: Optional[str] = None,
gt_expr: str = "GT",
f_stat_cutoff: float = 0.5,
aaf_threshold: float = 0.001,
) -> hl.Table:
"""
Impute sample sex based on X-chromosome heterozygosity and sex chromosome ploidy.
Return Table with the following fields:
- s (str): Sample
- chr20_mean_dp (float32): Sample's mean coverage over chromosome 20.
- chrX_mean_dp (float32): Sample's mean coverage over chromosome X.
- chrY_mean_dp (float32): Sample's mean coverage over chromosome Y.
- chrX_ploidy (float32): Sample's imputed ploidy over chromosome X.
- chrY_ploidy (float32): Sample's imputed ploidy over chromosome Y.
- f_stat (float64): Sample f-stat. Calculated using hl.impute_sex.
- n_called (int64): Number of variants with a genotype call. Calculated using hl.impute_sex.
- expected_homs (float64): Expected number of homozygotes. Calculated using hl.impute_sex.
- observed_homs (int64): Expected number of homozygotes. Calculated using hl.impute_sex.
- X_karyotype (str): Sample's chromosome X karyotype.
- Y_karyotype (str): Sample's chromosome Y karyotype.
- sex_karyotype (str): Sample's sex karyotype.
:param mtds: Input MatrixTable or VariantDataset
:param bool is_sparse: Whether input MatrixTable is in sparse data format
:param excluded_intervals: Optional table of intervals to exclude from the computation.
:param included_intervals: Optional table of intervals to use in the computation. REQUIRED for exomes.
:param normalization_contig: Which chromosome to use to normalize sex chromosome coverage. Used in determining sex chromosome ploidies.
:param reference_genome: Reference genome used for constructing interval list. Default: 'GRCh38'
:param sites_ht: Optional Table to use. If present, filters input MatrixTable to sites in this Table prior to imputing sex,
and pulls alternate allele frequency from this Table.
:param aaf_expr: Optional. Name of field in input MatrixTable with alternate allele frequency.
:param gt_expr: Name of entry field storing the genotype. Default: 'GT'
:param f_stat_cutoff: f-stat to roughly divide 'XX' from 'XY' samples. Assumes XX samples are below cutoff and XY are above cutoff.
:param float aaf_threshold: Minimum alternate allele frequency to be used in f-stat calculations.
:return: Table of samples and their imputed sex karyotypes.
"""
logger.info("Imputing sex chromosome ploidies...")
is_vds = isinstance(mtds, hl.vds.VariantDataset)
if is_vds:
if excluded_intervals is not None:
raise NotImplementedError(
"excluded_intervals is not used when imputing sex chromosome ploidy for VDS"
)
ploidy_ht = hl.vds.impute_sex_chromosome_ploidy(
mtds,
calling_intervals=included_intervals,
normalization_contig=normalization_contig,
)
ploidy_ht = ploidy_ht.rename(
{"x_ploidy": "chrX_ploidy", "y_ploidy": "chrY_ploidy"}
)
mt = mtds.variant_data
else:
mt = mtds
if is_sparse:
ploidy_ht = impute_sex_ploidy(
mt, excluded_intervals, included_intervals, normalization_contig
)
else:
raise NotImplementedError(
"Imputing sex ploidy does not exist yet for dense data."
)
x_contigs = get_reference_genome(mt.locus).x_contigs
logger.info("Filtering mt to biallelic SNPs in X contigs: %s", x_contigs)
if "was_split" in list(mt.row):
mt = mt.filter_rows((~mt.was_split) & hl.is_snp(mt.alleles[0], mt.alleles[1]))
else:
mt = mt.filter_rows(
(hl.len(mt.alleles) == 2) & hl.is_snp(mt.alleles[0], mt.alleles[1])
)
mt = hl.filter_intervals(
mt,
[
hl.parse_locus_interval(contig, reference_genome=reference_genome)
for contig in x_contigs
],
keep=True,
)
if sites_ht is not None:
if aaf_expr == None:
logger.warning(
"sites_ht was provided, but aaf_expr is missing. Assuming name of field with alternate allele frequency is 'AF'."
)
aaf_expr = "AF"
logger.info("Filtering to provided sites")
mt = mt.annotate_rows(**sites_ht[mt.row_key])
mt = mt.filter_rows(hl.is_defined(mt[aaf_expr]))
logger.info("Calculating inbreeding coefficient on chrX")
sex_ht = hl.impute_sex(
mt[gt_expr],
aaf_threshold=aaf_threshold,
male_threshold=f_stat_cutoff,
female_threshold=f_stat_cutoff,
aaf=aaf_expr,
)
logger.info("Annotating sex ht with sex chromosome ploidies")
sex_ht = sex_ht.annotate(**ploidy_ht[sex_ht.key])
logger.info("Inferring sex karyotypes")
x_ploidy_cutoffs, y_ploidy_cutoffs = get_ploidy_cutoffs(sex_ht, f_stat_cutoff)
sex_ht = sex_ht.annotate_globals(
x_ploidy_cutoffs=hl.struct(
upper_cutoff_X=x_ploidy_cutoffs[0],
lower_cutoff_XX=x_ploidy_cutoffs[1][0],
upper_cutoff_XX=x_ploidy_cutoffs[1][1],
lower_cutoff_XXX=x_ploidy_cutoffs[2],
),
y_ploidy_cutoffs=hl.struct(
lower_cutoff_Y=y_ploidy_cutoffs[0][0],
upper_cutoff_Y=y_ploidy_cutoffs[0][1],
lower_cutoff_YY=y_ploidy_cutoffs[1],
),
f_stat_cutoff=f_stat_cutoff,
)
return sex_ht.annotate(
**get_sex_expr(
sex_ht.chrX_ploidy, sex_ht.chrY_ploidy, x_ploidy_cutoffs, y_ploidy_cutoffs
)
)
def de_novo(mt: MatrixTable,
pedigree: Pedigree,
pop_frequency_prior,
*,
min_gq: int = 20,
min_p: float = 0.05,
max_parent_ab: float = 0.05,
min_child_ab: float = 0.20,
min_dp_ratio: float = 0.10) -> Table:
r"""Call putative *de novo* events from trio data.
.. include:: ../_templates/req_tstring.rst
.. include:: ../_templates/req_tvariant.rst
.. include:: ../_templates/req_biallelic.rst
Examples
--------
Call de novo events:
>>> pedigree = hl.Pedigree.read('data/trios.fam')
>>> priors = hl.import_table('data/gnomadFreq.tsv', impute=True)
>>> priors = priors.transmute(**hl.parse_variant(priors.Variant)).key_by('locus', 'alleles')
>>> de_novo_results = hl.de_novo(dataset, pedigree, pop_frequency_prior=priors[dataset.row_key].AF)
Notes
-----
This method assumes the GATK high-throughput sequencing fields exist:
`GT`, `AD`, `DP`, `GQ`, `PL`.
This method replicates the functionality of `Kaitlin Samocha's de novo
caller <https://github.com/ksamocha/de_novo_scripts>`__. The version
corresponding to git commit ``bde3e40`` is implemented in Hail with her
permission and assistance.
This method produces a :class:`.Table` with the following fields:
- `locus` (``locus``) -- Variant locus.
- `alleles` (``array<str>``) -- Variant alleles.
- `id` (``str``) -- Proband sample ID.
- `prior` (``float64``) -- Site frequency prior. It is the maximum of:
the computed dataset alternate allele frequency, the
`pop_frequency_prior` parameter, and the global prior
``1 / 3e7``.
- `proband` (``struct``) -- Proband column fields from `mt`.
- `father` (``struct``) -- Father column fields from `mt`.
- `mother` (``struct``) -- Mother column fields from `mt`.
- `proband_entry` (``struct``) -- Proband entry fields from `mt`.
- `father_entry` (``struct``) -- Father entry fields from `mt`.
- `proband_entry` (``struct``) -- Mother entry fields from `mt`.
- `is_female` (``bool``) -- ``True`` if proband is female.
- `p_de_novo` (``float64``) -- Unfiltered posterior probability
that the event is *de novo* rather than a missed heterozygous
event in a parent.
- `confidence` (``str``) Validation confidence. One of: ``'HIGH'``,
``'MEDIUM'``, ``'LOW'``.
The key of the table is ``['locus', 'alleles', 'id']``.
The model looks for de novo events in which both parents are homozygous
reference and the proband is a heterozygous. The model makes the simplifying
assumption that when this configuration ``x = (AA, AA, AB)`` of calls
occurs, exactly one of the following is true:
- ``d``: a de novo mutation occurred in the proband and all calls are
accurate.
- ``m``: at least one parental allele is actually heterozygous and
the proband call is accurate.
We can then estimate the posterior probability of a de novo mutation as:
.. math::
\mathrm{P_{\text{de novo}}} = \frac{\mathrm{P}(d\,|\,x)}{\mathrm{P}(d\,|\,x) + \mathrm{P}(m\,|\,x)}
Applying Bayes rule to the numerator and denominator yields
.. math::
\frac{\mathrm{P}(x\,|\,d)\,\mathrm{P}(d)}{\mathrm{P}(x\,|\,d)\,\mathrm{P}(d) +
\mathrm{P}(x\,|\,m)\,\mathrm{P}(m)}
The prior on de novo mutation is estimated from the rate in the literature:
.. math::
\mathrm{P}(d) = \frac{1 \text{mutation}}{30,000,000\, \text{bases}}
The prior used for at least one alternate allele between the parents
depends on the alternate allele frequency:
.. math::
\mathrm{P}(m) = 1 - (1 - AF)^4
The likelihoods :math:`\mathrm{P}(x\,|\,d)` and :math:`\mathrm{P}(x\,|\,m)`
are computed from the PL (genotype likelihood) fields using these
factorizations:
.. math::
\mathrm{P}(x = (AA, AA, AB) \,|\,d) = \Big(
&\mathrm{P}(x_{\mathrm{father}} = AA \,|\, \mathrm{father} = AA) \\
\cdot &\mathrm{P}(x_{\mathrm{mother}} = AA \,|\, \mathrm{mother} =
AA) \\ \cdot &\mathrm{P}(x_{\mathrm{proband}} = AB \,|\,
\mathrm{proband} = AB) \Big)
.. math::
\mathrm{P}(x = (AA, AA, AB) \,|\,m) = \Big( &
\mathrm{P}(x_{\mathrm{father}} = AA \,|\, \mathrm{father} = AB)
\cdot \mathrm{P}(x_{\mathrm{mother}} = AA \,|\, \mathrm{mother} =
AA) \\ + \, &\mathrm{P}(x_{\mathrm{father}} = AA \,|\,
\mathrm{father} = AA) \cdot \mathrm{P}(x_{\mathrm{mother}} = AA
\,|\, \mathrm{mother} = AB) \Big) \\ \cdot \,
&\mathrm{P}(x_{\mathrm{proband}} = AB \,|\, \mathrm{proband} = AB)
(Technically, the second factorization assumes there is exactly (rather
than at least) one alternate allele among the parents, which may be
justified on the grounds that it is typically the most likely case by far.)
While this posterior probability is a good metric for grouping putative de
novo mutations by validation likelihood, there exist error modes in
high-throughput sequencing data that are not appropriately accounted for by
the phred-scaled genotype likelihoods. To this end, a number of hard filters
are applied in order to assign validation likelihood.
These filters are different for SNPs and insertions/deletions. In the below
rules, the following variables are used:
- ``DR`` refers to the ratio of the read depth in the proband to the
combined read depth in the parents.
- ``AB`` refers to the read allele balance of the proband (number of
alternate reads divided by total reads).
- ``AC`` refers to the count of alternate alleles across all individuals
in the dataset at the site.
- ``p`` refers to :math:`\mathrm{P_{\text{de novo}}}`.
- ``min_p`` refers to the ``min_p`` function parameter.
HIGH-quality SNV:
.. code-block:: text
p > 0.99 && AB > 0.3 && DR > 0.2
or
p > 0.99 && AB > 0.3 && AC == 1
MEDIUM-quality SNV:
.. code-block:: text
p > 0.5 && AB > 0.3
or
p > 0.5 && AB > 0.2 && AC == 1
LOW-quality SNV:
.. code-block:: text
p > min_p && AB > 0.2
HIGH-quality indel:
.. code-block:: text
p > 0.99 && AB > 0.3 && DR > 0.2
or
p > 0.99 && AB > 0.3 && AC == 1
MEDIUM-quality indel:
.. code-block:: text
p > 0.5 && AB > 0.3
or
p > 0.5 && AB > 0.2 and AC == 1
LOW-quality indel:
.. code-block:: text
p > min_p && AB > 0.2
Additionally, de novo candidates are not considered if the proband GQ is
smaller than the ``min_gq`` parameter, if the proband allele balance is
lower than the ``min_child_ab`` parameter, if the depth ratio between the
proband and parents is smaller than the ``min_depth_ratio`` parameter, or if
the allele balance in a parent is above the ``max_parent_ab`` parameter.
Parameters
----------
mt : :class:`.MatrixTable`
High-throughput sequencing dataset.
pedigree : :class:`.Pedigree`
Sample pedigree.
pop_frequency_prior : :class:`.Float64Expression`
Expression for population alternate allele frequency prior.
min_gq
Minimum proband GQ to be considered for *de novo* calling.
min_p
Minimum posterior probability to be considered for *de novo* calling.
max_parent_ab
Maximum parent allele balance.
min_child_ab
Minimum proband allele balance/
min_dp_ratio
Minimum ratio between proband read depth and parental read depth.
Returns
-------
:class:`.Table`
"""
DE_NOVO_PRIOR = 1 / 30000000
MIN_POP_PRIOR = 100 / 30000000
required_entry_fields = {'GT', 'AD', 'DP', 'GQ', 'PL'}
missing_fields = required_entry_fields - set(mt.entry)
if missing_fields:
raise ValueError(f"'de_novo': expected 'MatrixTable' to have at least {required_entry_fields}, "
f"missing {missing_fields}")
mt = mt.annotate_rows(__prior=pop_frequency_prior,
__alt_alleles=hl.agg.sum(mt.GT.n_alt_alleles()),
__total_alleles=2 * hl.agg.sum(hl.is_defined(mt.GT)))
# subtract 1 from __alt_alleles to correct for the observed genotype
mt = mt.annotate_rows(__site_freq=hl.max((mt.__alt_alleles - 1) / mt.__total_alleles, mt.__prior, MIN_POP_PRIOR))
mt = require_biallelic(mt, 'de_novo')
# FIXME check that __site_freq is between 0 and 1 when possible in expr
tm = trio_matrix(mt, pedigree, complete_trios=True)
autosomal = tm.locus.in_autosome_or_par() | (tm.locus.in_x_nonpar() & tm.is_female)
hemi_x = tm.locus.in_x_nonpar() & ~tm.is_female
hemi_y = tm.locus.in_y_nonpar() & ~tm.is_female
hemi_mt = tm.locus.in_mito() & tm.is_female
is_snp = hl.is_snp(tm.alleles[0], tm.alleles[1])
n_alt_alleles = tm.__alt_alleles
prior = tm.__site_freq
het_hom_hom = tm.proband_entry.GT.is_het() & tm.father_entry.GT.is_hom_ref() & tm.mother_entry.GT.is_hom_ref()
kid_ad_fail = tm.proband_entry.AD[1] / hl.sum(tm.proband_entry.AD) < min_child_ab
failure = hl.null(hl.tstruct(p_de_novo=hl.tfloat64, confidence=hl.tstr))
kid = tm.proband_entry
dad = tm.father_entry
mom = tm.mother_entry
kid_linear_pl = 10 ** (-kid.PL / 10)
kid_pp = hl.bind(lambda x: x / hl.sum(x), kid_linear_pl)
dad_linear_pl = 10 ** (-dad.PL / 10)
dad_pp = hl.bind(lambda x: x / hl.sum(x), dad_linear_pl)
mom_linear_pl = 10 ** (-mom.PL / 10)
mom_pp = hl.bind(lambda x: x / hl.sum(x), mom_linear_pl)
kid_ad_ratio = kid.AD[1] / hl.sum(kid.AD)
dp_ratio = kid.DP / (dad.DP + mom.DP)
def call_auto(kid_pp, dad_pp, mom_pp, kid_ad_ratio):
p_data_given_dn = dad_pp[0] * mom_pp[0] * kid_pp[1] * DE_NOVO_PRIOR
p_het_in_parent = 1 - (1 - prior) ** 4
p_data_given_missed_het = (dad_pp[1] * mom_pp[0] + dad_pp[0] * mom_pp[1]) * kid_pp[1] * p_het_in_parent
p_de_novo = p_data_given_dn / (p_data_given_dn + p_data_given_missed_het)
def solve(p_de_novo):
return (
hl.case()
.when(kid.GQ < min_gq, failure)
.when((kid.DP / (dad.DP + mom.DP) < min_dp_ratio) |
~(kid_ad_ratio >= min_child_ab), failure)
.when((hl.sum(mom.AD) == 0) | (hl.sum(dad.AD) == 0), failure)
.when((mom.AD[1] / hl.sum(mom.AD) > max_parent_ab) |
(dad.AD[1] / hl.sum(dad.AD) > max_parent_ab), failure)
.when(p_de_novo < min_p, failure)
.when(~is_snp, hl.case()
.when((p_de_novo > 0.99) & (kid_ad_ratio > 0.3) & (n_alt_alleles == 1),
hl.struct(p_de_novo=p_de_novo, confidence='HIGH'))
.when((p_de_novo > 0.5) & (kid_ad_ratio > 0.3) & (n_alt_alleles <= 5),
hl.struct(p_de_novo=p_de_novo, confidence='MEDIUM'))
.when((p_de_novo > 0.05) & (kid_ad_ratio > 0.2),
hl.struct(p_de_novo=p_de_novo, confidence='LOW'))
.or_missing())
.default(hl.case()
.when(((p_de_novo > 0.99) & (kid_ad_ratio > 0.3) & (dp_ratio > 0.2)) |
((p_de_novo > 0.99) & (kid_ad_ratio > 0.3) & (n_alt_alleles == 1)) |
((p_de_novo > 0.5) & (kid_ad_ratio > 0.3) & (n_alt_alleles < 10) & (kid.DP > 10)),
hl.struct(p_de_novo=p_de_novo, confidence='HIGH'))
.when((p_de_novo > 0.5) & ((kid_ad_ratio > 0.3) | (n_alt_alleles == 1)),
hl.struct(p_de_novo=p_de_novo, confidence='MEDIUM'))
.when((p_de_novo > 0.05) & (kid_ad_ratio > 0.2),
hl.struct(p_de_novo=p_de_novo, confidence='LOW'))
.or_missing()
)
)
return hl.bind(solve, p_de_novo)
def call_hemi(kid_pp, parent, parent_pp, kid_ad_ratio):
p_data_given_dn = parent_pp[0] * kid_pp[1] * DE_NOVO_PRIOR
p_het_in_parent = 1 - (1 - prior) ** 4
p_data_given_missed_het = (parent_pp[1] + parent_pp[2]) * kid_pp[2] * p_het_in_parent
p_de_novo = p_data_given_dn / (p_data_given_dn + p_data_given_missed_het)
def solve(p_de_novo):
return (
hl.case()
.when(kid.GQ < min_gq, failure)
.when((kid.DP / (parent.DP) < min_dp_ratio) |
(kid_ad_ratio < min_child_ab), failure)
.when((hl.sum(parent.AD) == 0), failure)
.when(parent.AD[1] / hl.sum(parent.AD) > max_parent_ab, failure)
.when(p_de_novo < min_p, failure)
.when(~is_snp, hl.case()
.when((p_de_novo > 0.99) & (kid_ad_ratio > 0.3) & (n_alt_alleles == 1),
hl.struct(p_de_novo=p_de_novo, confidence='HIGH'))
.when((p_de_novo > 0.5) & (kid_ad_ratio > 0.3) & (n_alt_alleles <= 5),
hl.struct(p_de_novo=p_de_novo, confidence='MEDIUM'))
.when((p_de_novo > 0.05) & (kid_ad_ratio > 0.3),
hl.struct(p_de_novo=p_de_novo, confidence='LOW'))
.or_missing())
.default(hl.case()
.when(((p_de_novo > 0.99) & (kid_ad_ratio > 0.3) & (dp_ratio > 0.2)) |
((p_de_novo > 0.99) & (kid_ad_ratio > 0.3) & (n_alt_alleles == 1)) |
((p_de_novo > 0.5) & (kid_ad_ratio > 0.3) & (n_alt_alleles < 10) & (kid.DP > 10)),
hl.struct(p_de_novo=p_de_novo, confidence='HIGH'))
.when((p_de_novo > 0.5) & ((kid_ad_ratio > 0.3) | (n_alt_alleles == 1)),
hl.struct(p_de_novo=p_de_novo, confidence='MEDIUM'))
.when((p_de_novo > 0.05) & (kid_ad_ratio > 0.2),
hl.struct(p_de_novo=p_de_novo, confidence='LOW'))
.or_missing()
)
)
return hl.bind(solve, p_de_novo)
de_novo_call = (
hl.case()
.when(~het_hom_hom | kid_ad_fail, failure)
.when(autosomal, hl.bind(call_auto, kid_pp, dad_pp, mom_pp, kid_ad_ratio))
.when(hemi_x | hemi_mt, hl.bind(call_hemi, kid_pp, mom, mom_pp, kid_ad_ratio))
.when(hemi_y, hl.bind(call_hemi, kid_pp, dad, dad_pp, kid_ad_ratio))
.or_missing()
)
tm = tm.annotate_entries(__call=de_novo_call)
tm = tm.filter_entries(hl.is_defined(tm.__call))
entries = tm.entries()
return (entries.select('__site_freq',
'proband',
'father',
'mother',
'proband_entry',
'father_entry',
'mother_entry',
'is_female',
**entries.__call)
.rename({'__site_freq': 'prior'}))
def mendel_errors(call, pedigree) -> Tuple[Table, Table, Table, Table]:
r"""Find Mendel errors; count per variant, individual and nuclear family.
.. include:: ../_templates/req_tstring.rst
.. include:: ../_templates/req_tvariant.rst
.. include:: ../_templates/req_biallelic.rst
Examples
--------
Find all violations of Mendelian inheritance in each (dad, mom, kid) trio in
a pedigree and return four tables (all errors, errors by family, errors by
individual, errors by variant):
>>> ped = hl.Pedigree.read('data/trios.fam')
>>> all_errors, per_fam, per_sample, per_variant = hl.mendel_errors(dataset['GT'], ped)
Export all mendel errors to a text file:
>>> all_errors.export('output/all_mendel_errors.tsv')
Annotate columns with the number of Mendel errors:
>>> annotated_samples = dataset.annotate_cols(mendel=per_sample[dataset.s])
Annotate rows with the number of Mendel errors:
>>> annotated_variants = dataset.annotate_rows(mendel=per_variant[dataset.locus, dataset.alleles])
Notes
-----
The example above returns four tables, which contain Mendelian violations
grouped in various ways. These tables are modeled after the `PLINK mendel
formats <https://www.cog-genomics.org/plink2/formats#mendel>`_, resembling
the ``.mendel``, ``.fmendel``, ``.imendel``, and ``.lmendel`` formats,
respectively.
**First table:** all Mendel errors. This table contains one row per Mendel
error, keyed by the variant and proband id.
- `locus` (:class:`.tlocus`) -- Variant locus, key field.
- `alleles` (:class:`.tarray` of :py:data:`.tstr`) -- Variant alleles, key field.
- (column key of `dataset`) (:py:data:`.tstr`) -- Proband ID, key field.
- `fam_id` (:py:data:`.tstr`) -- Family ID.
- `mendel_code` (:py:data:`.tint32`) -- Mendel error code, see below.
**Second table:** errors per nuclear family. This table contains one row
per nuclear family, keyed by the parents.
- `pat_id` (:py:data:`.tstr`) -- Paternal ID. (key field)
- `mat_id` (:py:data:`.tstr`) -- Maternal ID. (key field)
- `fam_id` (:py:data:`.tstr`) -- Family ID.
- `children` (:py:data:`.tint32`) -- Number of children in this nuclear family.
- `errors` (:py:data:`.tint64`) -- Number of Mendel errors in this nuclear family.
- `snp_errors` (:py:data:`.tint64`) -- Number of Mendel errors at SNPs in this
nuclear family.
**Third table:** errors per individual. This table contains one row per
individual. Each error is counted toward the proband, father, and mother
according to the `Implicated` in the table below.
- (column key of `dataset`) (:py:data:`.tstr`) -- Sample ID (key field).
- `fam_id` (:py:data:`.tstr`) -- Family ID.
- `errors` (:py:data:`.tint64`) -- Number of Mendel errors involving this
individual.
- `snp_errors` (:py:data:`.tint64`) -- Number of Mendel errors involving this
individual at SNPs.
**Fourth table:** errors per variant.
- `locus` (:class:`.tlocus`) -- Variant locus, key field.
- `alleles` (:class:`.tarray` of :py:data:`.tstr`) -- Variant alleles, key field.
- `errors` (:py:data:`.tint64`) -- Number of Mendel errors in this variant.
This method only considers complete trios (two parents and proband with
defined sex). The code of each Mendel error is determined by the table
below, extending the
`Plink classification <https://www.cog-genomics.org/plink2/basic_stats#mendel>`__.
In the table, the copy state of a locus with respect to a trio is defined
as follows, where PAR is the `pseudoautosomal region
<https://en.wikipedia.org/wiki/Pseudoautosomal_region>`__ (PAR) of X and Y
defined by the reference genome and the autosome is defined by
:meth:`~hail.genetics.Locus.in_autosome`.
- Auto -- in autosome or in PAR or female child
- HemiX -- in non-PAR of X and male child
- HemiY -- in non-PAR of Y and male child
`Any` refers to the set \{ HomRef, Het, HomVar, NoCall \} and `~`
denotes complement in this set.
+------+---------+---------+--------+----------------------------+
| Code | Dad | Mom | Kid | Copy State | Implicated |
+======+=========+=========+========+============+===============+
| 1 | HomVar | HomVar | Het | Auto | Dad, Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 2 | HomRef | HomRef | Het | Auto | Dad, Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 3 | HomRef | ~HomRef | HomVar | Auto | Dad, Kid |
+------+---------+---------+--------+------------+---------------+
| 4 | ~HomRef | HomRef | HomVar | Auto | Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 5 | HomRef | HomRef | HomVar | Auto | Kid |
+------+---------+---------+--------+------------+---------------+
| 6 | HomVar | ~HomVar | HomRef | Auto | Dad, Kid |
+------+---------+---------+--------+------------+---------------+
| 7 | ~HomVar | HomVar | HomRef | Auto | Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 8 | HomVar | HomVar | HomRef | Auto | Kid |
+------+---------+---------+--------+------------+---------------+
| 9 | Any | HomVar | HomRef | HemiX | Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 10 | Any | HomRef | HomVar | HemiX | Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 11 | HomVar | Any | HomRef | HemiY | Dad, Kid |
+------+---------+---------+--------+------------+---------------+
| 12 | HomRef | Any | HomVar | HemiY | Dad, Kid |
+------+---------+---------+--------+------------+---------------+
See Also
--------
:func:`.mendel_error_code`
Parameters
----------
dataset : :class:`.MatrixTable`
pedigree : :class:`.Pedigree`
Returns
-------
(:class:`.Table`, :class:`.Table`, :class:`.Table`, :class:`.Table`)
"""
source = call._indices.source
if not isinstance(source, MatrixTable):
raise ValueError(
"'mendel_errors': expected 'call' to be an expression of 'MatrixTable', found {}"
.format("expression of '{}'".format(source.__class__)
if source is not None else 'scalar expression'))
source = source.select_entries(__GT=call)
dataset = require_biallelic(source, 'mendel_errors')
tm = trio_matrix(dataset, pedigree, complete_trios=True)
tm = tm.select_entries(mendel_code=hl.mendel_error_code(
tm.locus, tm.is_female, tm.father_entry['__GT'],
tm.mother_entry['__GT'], tm.proband_entry['__GT']))
ck_name = next(iter(source.col_key))
tm = tm.filter_entries(hl.is_defined(tm.mendel_code))
tm = tm.rename({'id': ck_name})
entries = tm.entries()
table1 = entries.select('fam_id', 'mendel_code')
fam_counts = (entries.group_by(
pat_id=entries.father[ck_name],
mat_id=entries.mother[ck_name]).partition_hint(
min(entries.n_partitions(), 8)).aggregate(
children=hl.len(hl.agg.collect_as_set(entries[ck_name])),
errors=hl.agg.count_where(hl.is_defined(entries.mendel_code)),
snp_errors=hl.agg.count_where(
hl.is_snp(entries.alleles[0], entries.alleles[1])
& hl.is_defined(entries.mendel_code))))
table2 = tm.key_cols_by().cols()
table2 = table2.select(pat_id=table2.father[ck_name],
mat_id=table2.mother[ck_name],
fam_id=table2.fam_id,
**fam_counts[table2.father[ck_name],
table2.mother[ck_name]])
table2 = table2.key_by('pat_id', 'mat_id').distinct()
table2 = table2.annotate(errors=hl.or_else(table2.errors, hl.int64(0)),
snp_errors=hl.or_else(table2.snp_errors,
hl.int64(0)))
# in implicated, idx 0 is dad, idx 1 is mom, idx 2 is child
implicated = hl.literal(
[
[0, 0, 0], # dummy
[1, 1, 1],
[1, 1, 1],
[1, 0, 1],
[0, 1, 1],
[0, 0, 1],
[1, 0, 1],
[0, 1, 1],
[0, 0, 1],
[0, 1, 1],
[0, 1, 1],
[1, 0, 1],
[1, 0, 1],
],
dtype=hl.tarray(hl.tarray(hl.tint64)))
table3 = tm.annotate_cols(
all_errors=hl.or_else(hl.agg.array_sum(implicated[tm.mendel_code]),
[0, 0, 0]),
snp_errors=hl.or_else(
hl.agg.filter(hl.is_snp(tm.alleles[0], tm.alleles[1]),
hl.agg.array_sum(implicated[tm.mendel_code])),
[0, 0, 0])).key_cols_by().cols()
table3 = table3.select(xs=[
hl.struct(
**{
ck_name: table3.father[ck_name],
'fam_id': table3.fam_id,
'errors': table3.all_errors[0],
'snp_errors': table3.snp_errors[0]
}),
hl.struct(
**{
ck_name: table3.mother[ck_name],
'fam_id': table3.fam_id,
'errors': table3.all_errors[1],
'snp_errors': table3.snp_errors[1]
}),
hl.struct(
**{
ck_name: table3.proband[ck_name],
'fam_id': table3.fam_id,
'errors': table3.all_errors[2],
'snp_errors': table3.snp_errors[2]
}),
])
table3 = table3.explode('xs')
table3 = table3.select(**table3.xs)
table3 = (table3.group_by(ck_name, 'fam_id').aggregate(
errors=hl.agg.sum(table3.errors),
snp_errors=hl.agg.sum(table3.snp_errors)).key_by(ck_name))
table4 = tm.select_rows(
errors=hl.agg.count_where(hl.is_defined(tm.mendel_code))).rows()
return table1, table2, table3, table4
def de_novo(mt: MatrixTable,
pedigree: Pedigree,
pop_frequency_prior,
*,
min_gq: int = 20,
min_p: float = 0.05,
max_parent_ab: float = 0.05,
min_child_ab: float = 0.20,
min_dp_ratio: float = 0.10) -> Table:
r"""Call putative *de novo* events from trio data.
.. include:: ../_templates/req_tstring.rst
.. include:: ../_templates/req_tvariant.rst
.. include:: ../_templates/req_biallelic.rst
Examples
--------
Call de novo events:
>>> pedigree = hl.Pedigree.read('data/trios.fam')
>>> priors = hl.import_table('data/gnomadFreq.tsv', impute=True)
>>> priors = priors.transmute(**hl.parse_variant(priors.Variant)).key_by('locus', 'alleles')
>>> de_novo_results = hl.de_novo(dataset, pedigree, pop_frequency_prior=priors[dataset.row_key].AF)
Notes
-----
This method assumes the GATK high-throughput sequencing fields exist:
`GT`, `AD`, `DP`, `GQ`, `PL`.
This method replicates the functionality of `Kaitlin Samocha's de novo
caller <https://github.com/ksamocha/de_novo_scripts>`__. The version
corresponding to git commit ``bde3e40`` is implemented in Hail with her
permission and assistance.
This method produces a :class:`.Table` with the following fields:
- `locus` (``locus``) -- Variant locus.
- `alleles` (``array<str>``) -- Variant alleles.
- `id` (``str``) -- Proband sample ID.
- `prior` (``float64``) -- Site frequency prior. It is the maximum of:
the computed dataset alternate allele frequency, the
`pop_frequency_prior` parameter, and the global prior
``1 / 3e7``.
- `proband` (``struct``) -- Proband column fields from `mt`.
- `father` (``struct``) -- Father column fields from `mt`.
- `mother` (``struct``) -- Mother column fields from `mt`.
- `proband_entry` (``struct``) -- Proband entry fields from `mt`.
- `father_entry` (``struct``) -- Father entry fields from `mt`.
- `proband_entry` (``struct``) -- Mother entry fields from `mt`.
- `is_female` (``bool``) -- ``True`` if proband is female.
- `p_de_novo` (``float64``) -- Unfiltered posterior probability
that the event is *de novo* rather than a missed heterozygous
event in a parent.
- `confidence` (``str``) Validation confidence. One of: ``'HIGH'``,
``'MEDIUM'``, ``'LOW'``.
The key of the table is ``['locus', 'alleles', 'id']``.
The model looks for de novo events in which both parents are homozygous
reference and the proband is a heterozygous. The model makes the simplifying
assumption that when this configuration ``x = (AA, AA, AB)`` of calls
occurs, exactly one of the following is true:
- ``d``: a de novo mutation occurred in the proband and all calls are
accurate.
- ``m``: at least one parental allele is actually heterozygous and
the proband call is accurate.
We can then estimate the posterior probability of a de novo mutation as:
.. math::
\mathrm{P_{\text{de novo}}} = \frac{\mathrm{P}(d\,|\,x)}{\mathrm{P}(d\,|\,x) + \mathrm{P}(m\,|\,x)}
Applying Bayes rule to the numerator and denominator yields
.. math::
\frac{\mathrm{P}(x\,|\,d)\,\mathrm{P}(d)}{\mathrm{P}(x\,|\,d)\,\mathrm{P}(d) +
\mathrm{P}(x\,|\,m)\,\mathrm{P}(m)}
The prior on de novo mutation is estimated from the rate in the literature:
.. math::
\mathrm{P}(d) = \frac{1 \text{mutation}}{30,000,000\, \text{bases}}
The prior used for at least one alternate allele between the parents
depends on the alternate allele frequency:
.. math::
\mathrm{P}(m) = 1 - (1 - AF)^4
The likelihoods :math:`\mathrm{P}(x\,|\,d)` and :math:`\mathrm{P}(x\,|\,m)`
are computed from the PL (genotype likelihood) fields using these
factorizations:
.. math::
\mathrm{P}(x = (AA, AA, AB) \,|\,d) = \Big(
&\mathrm{P}(x_{\mathrm{father}} = AA \,|\, \mathrm{father} = AA) \\
\cdot &\mathrm{P}(x_{\mathrm{mother}} = AA \,|\, \mathrm{mother} =
AA) \\ \cdot &\mathrm{P}(x_{\mathrm{proband}} = AB \,|\,
\mathrm{proband} = AB) \Big)
.. math::
\mathrm{P}(x = (AA, AA, AB) \,|\,m) = \Big( &
\mathrm{P}(x_{\mathrm{father}} = AA \,|\, \mathrm{father} = AB)
\cdot \mathrm{P}(x_{\mathrm{mother}} = AA \,|\, \mathrm{mother} =
AA) \\ + \, &\mathrm{P}(x_{\mathrm{father}} = AA \,|\,
\mathrm{father} = AA) \cdot \mathrm{P}(x_{\mathrm{mother}} = AA
\,|\, \mathrm{mother} = AB) \Big) \\ \cdot \,
&\mathrm{P}(x_{\mathrm{proband}} = AB \,|\, \mathrm{proband} = AB)
(Technically, the second factorization assumes there is exactly (rather
than at least) one alternate allele among the parents, which may be
justified on the grounds that it is typically the most likely case by far.)
While this posterior probability is a good metric for grouping putative de
novo mutations by validation likelihood, there exist error modes in
high-throughput sequencing data that are not appropriately accounted for by
the phred-scaled genotype likelihoods. To this end, a number of hard filters
are applied in order to assign validation likelihood.
These filters are different for SNPs and insertions/deletions. In the below
rules, the following variables are used:
- ``DR`` refers to the ratio of the read depth in the proband to the
combined read depth in the parents.
- ``AB`` refers to the read allele balance of the proband (number of
alternate reads divided by total reads).
- ``AC`` refers to the count of alternate alleles across all individuals
in the dataset at the site.
- ``p`` refers to :math:`\mathrm{P_{\text{de novo}}}`.
- ``min_p`` refers to the ``min_p`` function parameter.
HIGH-quality SNV:
.. code-block:: text
p > 0.99 && AB > 0.3 && DR > 0.2
or
p > 0.99 && AB > 0.3 && AC == 1
MEDIUM-quality SNV:
.. code-block:: text
p > 0.5 && AB > 0.3
or
p > 0.5 && AB > 0.2 && AC == 1
LOW-quality SNV:
.. code-block:: text
p > min_p && AB > 0.2
HIGH-quality indel:
.. code-block:: text
p > 0.99 && AB > 0.3 && DR > 0.2
or
p > 0.99 && AB > 0.3 && AC == 1
MEDIUM-quality indel:
.. code-block:: text
p > 0.5 && AB > 0.3
or
p > 0.5 && AB > 0.2 and AC == 1
LOW-quality indel:
.. code-block:: text
p > min_p && AB > 0.2
Additionally, de novo candidates are not considered if the proband GQ is
smaller than the ``min_gq`` parameter, if the proband allele balance is
lower than the ``min_child_ab`` parameter, if the depth ratio between the
proband and parents is smaller than the ``min_depth_ratio`` parameter, or if
the allele balance in a parent is above the ``max_parent_ab`` parameter.
Parameters
----------
mt : :class:`.MatrixTable`
High-throughput sequencing dataset.
pedigree : :class:`.Pedigree`
Sample pedigree.
pop_frequency_prior : :class:`.Float64Expression`
Expression for population alternate allele frequency prior.
min_gq
Minimum proband GQ to be considered for *de novo* calling.
min_p
Minimum posterior probability to be considered for *de novo* calling.
max_parent_ab
Maximum parent allele balance.
min_child_ab
Minimum proband allele balance/
min_dp_ratio
Minimum ratio between proband read depth and parental read depth.
Returns
-------
:class:`.Table`
"""
DE_NOVO_PRIOR = 1 / 30000000
MIN_POP_PRIOR = 100 / 30000000
required_entry_fields = {'GT', 'AD', 'DP', 'GQ', 'PL'}
missing_fields = required_entry_fields - set(mt.entry)
if missing_fields:
raise ValueError(
f"'de_novo': expected 'MatrixTable' to have at least {required_entry_fields}, "
f"missing {missing_fields}")
mt = mt.annotate_rows(__prior=pop_frequency_prior,
__alt_alleles=hl.agg.sum(mt.GT.n_alt_alleles()),
__total_alleles=2 * hl.agg.sum(hl.is_defined(mt.GT)))
# subtract 1 from __alt_alleles to correct for the observed genotype
mt = mt.annotate_rows(
__site_freq=hl.max((mt.__alt_alleles - 1) /
mt.__total_alleles, mt.__prior, MIN_POP_PRIOR))
mt = require_biallelic(mt, 'de_novo')
# FIXME check that __site_freq is between 0 and 1 when possible in expr
tm = trio_matrix(mt, pedigree, complete_trios=True)
autosomal = tm.locus.in_autosome_or_par() | (tm.locus.in_x_nonpar()
& tm.is_female)
hemi_x = tm.locus.in_x_nonpar() & ~tm.is_female
hemi_y = tm.locus.in_y_nonpar() & ~tm.is_female
hemi_mt = tm.locus.in_mito() & tm.is_female
is_snp = hl.is_snp(tm.alleles[0], tm.alleles[1])
n_alt_alleles = tm.__alt_alleles
prior = tm.__site_freq
het_hom_hom = tm.proband_entry.GT.is_het() & tm.father_entry.GT.is_hom_ref(
) & tm.mother_entry.GT.is_hom_ref()
kid_ad_fail = tm.proband_entry.AD[1] / hl.sum(
tm.proband_entry.AD) < min_child_ab
failure = hl.null(hl.tstruct(p_de_novo=hl.tfloat64, confidence=hl.tstr))
kid = tm.proband_entry
dad = tm.father_entry
mom = tm.mother_entry
kid_linear_pl = 10**(-kid.PL / 10)
kid_pp = hl.bind(lambda x: x / hl.sum(x), kid_linear_pl)
dad_linear_pl = 10**(-dad.PL / 10)
dad_pp = hl.bind(lambda x: x / hl.sum(x), dad_linear_pl)
mom_linear_pl = 10**(-mom.PL / 10)
mom_pp = hl.bind(lambda x: x / hl.sum(x), mom_linear_pl)
kid_ad_ratio = kid.AD[1] / hl.sum(kid.AD)
dp_ratio = kid.DP / (dad.DP + mom.DP)
def call_auto(kid_pp, dad_pp, mom_pp, kid_ad_ratio):
p_data_given_dn = dad_pp[0] * mom_pp[0] * kid_pp[1] * DE_NOVO_PRIOR
p_het_in_parent = 1 - (1 - prior)**4
p_data_given_missed_het = (dad_pp[1] * mom_pp[0] + dad_pp[0] *
mom_pp[1]) * kid_pp[1] * p_het_in_parent
p_de_novo = p_data_given_dn / (p_data_given_dn +
p_data_given_missed_het)
def solve(p_de_novo):
return (hl.case().when(kid.GQ < min_gq, failure).when(
(kid.DP / (dad.DP + mom.DP) < min_dp_ratio)
| ~(kid_ad_ratio >= min_child_ab), failure).when(
(hl.sum(mom.AD) == 0) | (hl.sum(dad.AD) == 0),
failure).when(
(mom.AD[1] / hl.sum(mom.AD) > max_parent_ab) |
(dad.AD[1] / hl.sum(dad.AD) > max_parent_ab),
failure).when(p_de_novo < min_p, failure).when(
~is_snp,
hl.case().when(
(p_de_novo > 0.99) & (kid_ad_ratio > 0.3) &
(n_alt_alleles == 1),
hl.struct(p_de_novo=p_de_novo,
confidence='HIGH')).when(
(p_de_novo > 0.5) &
(kid_ad_ratio > 0.3) &
(n_alt_alleles <= 5),
hl.struct(
p_de_novo=p_de_novo,
confidence='MEDIUM')).when(
(p_de_novo > 0.05) &
(kid_ad_ratio > 0.2),
hl.struct(
p_de_novo=p_de_novo,
confidence='LOW')).
or_missing()).default(hl.case().when(
((p_de_novo > 0.99) & (kid_ad_ratio > 0.3) &
(dp_ratio > 0.2)) | ((p_de_novo > 0.99) &
(kid_ad_ratio > 0.3) &
(n_alt_alleles == 1)) |
((p_de_novo > 0.5) & (kid_ad_ratio > 0.3) &
(n_alt_alleles < 10) & (kid.DP > 10)),
hl.struct(p_de_novo=p_de_novo,
confidence='HIGH')).when(
(p_de_novo > 0.5) &
((kid_ad_ratio > 0.3) |
(n_alt_alleles == 1)),
hl.struct(
p_de_novo=p_de_novo,
confidence='MEDIUM')).when(
(p_de_novo > 0.05) &
(kid_ad_ratio > 0.2),
hl.struct(
p_de_novo=p_de_novo,
confidence='LOW')).
or_missing()))
return hl.bind(solve, p_de_novo)
def call_hemi(kid_pp, parent, parent_pp, kid_ad_ratio):
p_data_given_dn = parent_pp[0] * kid_pp[1] * DE_NOVO_PRIOR
p_het_in_parent = 1 - (1 - prior)**4
p_data_given_missed_het = (parent_pp[1] +
parent_pp[2]) * kid_pp[2] * p_het_in_parent
p_de_novo = p_data_given_dn / (p_data_given_dn +
p_data_given_missed_het)
def solve(p_de_novo):
return (hl.case().when(kid.GQ < min_gq, failure).when(
(kid.DP /
(parent.DP) < min_dp_ratio) | (kid_ad_ratio < min_child_ab),
failure).when((hl.sum(parent.AD) == 0), failure).when(
parent.AD[1] / hl.sum(parent.AD) > max_parent_ab,
failure).when(p_de_novo < min_p, failure).when(
~is_snp,
hl.case().when(
(p_de_novo > 0.99) & (kid_ad_ratio > 0.3) &
(n_alt_alleles == 1),
hl.struct(
p_de_novo=p_de_novo, confidence='HIGH')).when(
(p_de_novo > 0.5) & (kid_ad_ratio > 0.3) &
(n_alt_alleles <= 5),
hl.struct(p_de_novo=p_de_novo,
confidence='MEDIUM')).when(
(p_de_novo > 0.05) &
(kid_ad_ratio > 0.3),
hl.struct(
p_de_novo=p_de_novo,
confidence='LOW')).
or_missing()).default(
hl.case().when(
((p_de_novo > 0.99) & (kid_ad_ratio > 0.3) &
(dp_ratio > 0.2)) |
((p_de_novo > 0.99) & (kid_ad_ratio > 0.3) &
(n_alt_alleles == 1)) |
((p_de_novo > 0.5) & (kid_ad_ratio > 0.3) &
(n_alt_alleles < 10) & (kid.DP > 10)),
hl.struct(p_de_novo=p_de_novo,
confidence='HIGH')).when(
(p_de_novo > 0.5) &
((kid_ad_ratio > 0.3) |
(n_alt_alleles == 1)),
hl.struct(p_de_novo=p_de_novo,
confidence='MEDIUM')).
when((p_de_novo > 0.05) & (kid_ad_ratio > 0.2),
hl.struct(p_de_novo=p_de_novo,
confidence='LOW')).or_missing()))
return hl.bind(solve, p_de_novo)
de_novo_call = (hl.case().when(~het_hom_hom | kid_ad_fail, failure).when(
autosomal,
hl.bind(call_auto, kid_pp, dad_pp, mom_pp, kid_ad_ratio)).when(
hemi_x | hemi_mt,
hl.bind(call_hemi, kid_pp, mom, mom_pp, kid_ad_ratio)).when(
hemi_y, hl.bind(call_hemi, kid_pp, dad, dad_pp,
kid_ad_ratio)).or_missing())
tm = tm.annotate_entries(__call=de_novo_call)
tm = tm.filter_entries(hl.is_defined(tm.__call))
entries = tm.entries()
return (entries.select('__site_freq', 'proband', 'father', 'mother',
'proband_entry', 'father_entry', 'mother_entry',
'is_female',
**entries.__call).rename({'__site_freq': 'prior'}))
def mendel_errors(call, pedigree) -> Tuple[Table, Table, Table, Table]:
r"""Find Mendel errors; count per variant, individual and nuclear family.
.. include:: ../_templates/req_tstring.rst
.. include:: ../_templates/req_tvariant.rst
.. include:: ../_templates/req_biallelic.rst
Examples
--------
Find all violations of Mendelian inheritance in each (dad, mom, kid) trio in
a pedigree and return four tables (all errors, errors by family, errors by
individual, errors by variant):
>>> ped = hl.Pedigree.read('data/trios.fam')
>>> all_errors, per_fam, per_sample, per_variant = hl.mendel_errors(dataset['GT'], ped)
Export all mendel errors to a text file:
>>> all_errors.export('output/all_mendel_errors.tsv')
Annotate columns with the number of Mendel errors:
>>> annotated_samples = dataset.annotate_cols(mendel=per_sample[dataset.s])
Annotate rows with the number of Mendel errors:
>>> annotated_variants = dataset.annotate_rows(mendel=per_variant[dataset.locus, dataset.alleles])
Notes
-----
The example above returns four tables, which contain Mendelian violations
grouped in various ways. These tables are modeled after the `PLINK mendel
formats <https://www.cog-genomics.org/plink2/formats#mendel>`_, resembling
the ``.mendel``, ``.fmendel``, ``.imendel``, and ``.lmendel`` formats,
respectively.
**First table:** all Mendel errors. This table contains one row per Mendel
error, keyed by the variant and proband id.
- `locus` (:class:`.tlocus`) -- Variant locus, key field.
- `alleles` (:class:`.tarray` of :py:data:`.tstr`) -- Variant alleles, key field.
- (column key of `dataset`) (:py:data:`.tstr`) -- Proband ID, key field.
- `fam_id` (:py:data:`.tstr`) -- Family ID.
- `mendel_code` (:py:data:`.tint32`) -- Mendel error code, see below.
**Second table:** errors per nuclear family. This table contains one row
per nuclear family, keyed by the parents.
- `pat_id` (:py:data:`.tstr`) -- Paternal ID. (key field)
- `mat_id` (:py:data:`.tstr`) -- Maternal ID. (key field)
- `fam_id` (:py:data:`.tstr`) -- Family ID.
- `children` (:py:data:`.tint32`) -- Number of children in this nuclear family.
- `errors` (:py:data:`.tint64`) -- Number of Mendel errors in this nuclear family.
- `snp_errors` (:py:data:`.tint64`) -- Number of Mendel errors at SNPs in this
nuclear family.
**Third table:** errors per individual. This table contains one row per
individual. Each error is counted toward the proband, father, and mother
according to the `Implicated` in the table below.
- (column key of `dataset`) (:py:data:`.tstr`) -- Sample ID (key field).
- `fam_id` (:py:data:`.tstr`) -- Family ID.
- `errors` (:py:data:`.tint64`) -- Number of Mendel errors involving this
individual.
- `snp_errors` (:py:data:`.tint64`) -- Number of Mendel errors involving this
individual at SNPs.
**Fourth table:** errors per variant.
- `locus` (:class:`.tlocus`) -- Variant locus, key field.
- `alleles` (:class:`.tarray` of :py:data:`.tstr`) -- Variant alleles, key field.
- `errors` (:py:data:`.tint64`) -- Number of Mendel errors in this variant.
This method only considers complete trios (two parents and proband with
defined sex). The code of each Mendel error is determined by the table
below, extending the
`Plink classification <https://www.cog-genomics.org/plink2/basic_stats#mendel>`__.
In the table, the copy state of a locus with respect to a trio is defined
as follows, where PAR is the `pseudoautosomal region
<https://en.wikipedia.org/wiki/Pseudoautosomal_region>`__ (PAR) of X and Y
defined by the reference genome and the autosome is defined by
:meth:`~hail.genetics.Locus.in_autosome`.
- Auto -- in autosome or in PAR or female child
- HemiX -- in non-PAR of X and male child
- HemiY -- in non-PAR of Y and male child
`Any` refers to the set \{ HomRef, Het, HomVar, NoCall \} and `~`
denotes complement in this set.
+------+---------+---------+--------+----------------------------+
| Code | Dad | Mom | Kid | Copy State | Implicated |
+======+=========+=========+========+============+===============+
| 1 | HomVar | HomVar | Het | Auto | Dad, Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 2 | HomRef | HomRef | Het | Auto | Dad, Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 3 | HomRef | ~HomRef | HomVar | Auto | Dad, Kid |
+------+---------+---------+--------+------------+---------------+
| 4 | ~HomRef | HomRef | HomVar | Auto | Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 5 | HomRef | HomRef | HomVar | Auto | Kid |
+------+---------+---------+--------+------------+---------------+
| 6 | HomVar | ~HomVar | HomRef | Auto | Dad, Kid |
+------+---------+---------+--------+------------+---------------+
| 7 | ~HomVar | HomVar | HomRef | Auto | Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 8 | HomVar | HomVar | HomRef | Auto | Kid |
+------+---------+---------+--------+------------+---------------+
| 9 | Any | HomVar | HomRef | HemiX | Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 10 | Any | HomRef | HomVar | HemiX | Mom, Kid |
+------+---------+---------+--------+------------+---------------+
| 11 | HomVar | Any | HomRef | HemiY | Dad, Kid |
+------+---------+---------+--------+------------+---------------+
| 12 | HomRef | Any | HomVar | HemiY | Dad, Kid |
+------+---------+---------+--------+------------+---------------+
See Also
--------
:func:`.mendel_error_code`
Parameters
----------
dataset : :class:`.MatrixTable`
pedigree : :class:`.Pedigree`
Returns
-------
(:class:`.Table`, :class:`.Table`, :class:`.Table`, :class:`.Table`)
"""
source = call._indices.source
if not isinstance(source, MatrixTable):
raise ValueError("'mendel_errors': expected 'call' to be an expression of 'MatrixTable', found {}".format(
"expression of '{}'".format(source.__class__) if source is not None else 'scalar expression'))
source = source.select_entries(__GT=call)
dataset = require_biallelic(source, 'mendel_errors')
tm = trio_matrix(dataset, pedigree, complete_trios=True)
tm = tm.select_entries(mendel_code=hl.mendel_error_code(
tm.locus,
tm.is_female,
tm.father_entry['__GT'],
tm.mother_entry['__GT'],
tm.proband_entry['__GT']
))
ck_name = next(iter(source.col_key))
tm = tm.filter_entries(hl.is_defined(tm.mendel_code))
tm = tm.rename({'id' : ck_name})
entries = tm.entries()
table1 = entries.select('fam_id', 'mendel_code')
fam_counts = (
entries
.group_by(pat_id=entries.father[ck_name], mat_id=entries.mother[ck_name])
.partition_hint(min(entries.n_partitions(), 8))
.aggregate(children=hl.len(hl.agg.collect_as_set(entries[ck_name])),
errors=hl.agg.count_where(hl.is_defined(entries.mendel_code)),
snp_errors=hl.agg.count_where(hl.is_snp(entries.alleles[0], entries.alleles[1]) &
hl.is_defined(entries.mendel_code)))
)
table2 = tm.key_cols_by().cols()
table2 = table2.select(pat_id=table2.father[ck_name],
mat_id=table2.mother[ck_name],
fam_id=table2.fam_id,
**fam_counts[table2.father[ck_name], table2.mother[ck_name]])
table2 = table2.key_by('pat_id', 'mat_id').distinct()
table2 = table2.annotate(errors=hl.or_else(table2.errors, hl.int64(0)),
snp_errors=hl.or_else(table2.snp_errors, hl.int64(0)))
# in implicated, idx 0 is dad, idx 1 is mom, idx 2 is child
implicated = hl.literal([
[0, 0, 0], # dummy
[1, 1, 1],
[1, 1, 1],
[1, 0, 1],
[0, 1, 1],
[0, 0, 1],
[1, 0, 1],
[0, 1, 1],
[0, 0, 1],
[0, 1, 1],
[0, 1, 1],
[1, 0, 1],
[1, 0, 1],
], dtype=hl.tarray(hl.tarray(hl.tint64)))
table3 = tm.annotate_cols(all_errors=hl.or_else(hl.agg.array_sum(implicated[tm.mendel_code]), [0, 0, 0]),
snp_errors=hl.or_else(
hl.agg.filter(hl.is_snp(tm.alleles[0], tm.alleles[1]),
hl.agg.array_sum(implicated[tm.mendel_code])),
[0, 0, 0])).key_cols_by().cols()
table3 = table3.select(xs=[
hl.struct(**{ck_name: table3.father[ck_name],
'fam_id': table3.fam_id,
'errors': table3.all_errors[0],
'snp_errors': table3.snp_errors[0]}),
hl.struct(**{ck_name: table3.mother[ck_name],
'fam_id': table3.fam_id,
'errors': table3.all_errors[1],
'snp_errors': table3.snp_errors[1]}),
hl.struct(**{ck_name: table3.proband[ck_name],
'fam_id': table3.fam_id,
'errors': table3.all_errors[2],
'snp_errors': table3.snp_errors[2]}),
])
table3 = table3.explode('xs')
table3 = table3.select(**table3.xs)
table3 = (table3.group_by(ck_name, 'fam_id')
.aggregate(errors=hl.agg.sum(table3.errors),
snp_errors=hl.agg.sum(table3.snp_errors))
.key_by(ck_name))
table4 = tm.select_rows(errors=hl.agg.count_where(hl.is_defined(tm.mendel_code))).rows()
return table1, table2, table3, table4
def matrix_table_rows_is_transition():
ht = hl.read_matrix_table(resource('profile.mt')).rows().key_by()
ht.select(is_snp = hl.is_snp(ht.alleles[0], ht.alleles[1]))._force_count()