def generate_final_rf_ht(
ht: hl.Table,
snp_cutoff: Union[int, float],
indel_cutoff: Union[int, float],
inbreeding_coeff_cutoff: float = INBREEDING_COEFF_HARD_CUTOFF,
determine_cutoff_from_bin: bool = False,
aggregated_bin_ht: Optional[hl.Table] = None,
bin_id: Optional[hl.expr.Int32Expression] = None,
) -> hl.Table:
"""
Prepares finalized RF model given an RF result table from `rf.apply_rf_model` and cutoffs for filtering.
If `determine_cutoff_from_bin` is True, `aggregated_bin_ht` must be supplied to determine the SNP and indel RF
probabilities to use as cutoffs from an aggregated quantile bin Table like one created by
`compute_grouped_binned_ht` in combination with `score_bin_agg`.
:param ht: RF result table from `rf.apply_rf_model` to prepare as the final RF Table
:param ac0_filter_expr: Expression that indicates if a variant should be filtered as allele count 0 (AC0)
:param ts_ac_filter_expr: Expression in `ht` that indicates if a variant is a transmitted singleton
:param mono_allelic_fiter_expr: Expression indicating if a variant is mono-allelic
:param snp_cutoff: RF probability or bin (if `determine_cutoff_from_bin` True) to use for SNP variant QC filter
:param indel_cutoff: RF probability or bin (if `determine_cutoff_from_bin` True) to use for indel variant QC filter
:param inbreeding_coeff_cutoff: InbreedingCoeff hard filter to use for variants
:param determine_cutoff_from_bin: If True RF probability will be determined using bin info in `aggregated_bin_ht`
:param aggregated_bin_ht: File with aggregate counts of variants based on quantile bins
:param bin_id: Name of bin to use in 'bin_id' column of `aggregated_bin_ht` to use to determine probability cutoff
:return: Finalized random forest Table annotated with variant filters
"""
# Determine SNP and indel RF cutoffs if given bin instead of RF probability
snp_cutoff_global = hl.struct(min_score=snp_cutoff)
indel_cutoff_global = hl.struct(min_score=indel_cutoff)
# Add filters to RF HT
filters = dict()
if ht.any(hl.is_missing(ht.rf_probability["TP"])):
raise ValueError("Missing RF probability!")
filters["RF"] = (
hl.is_snp(ht.alleles[0], ht.alleles[1])
& (ht.rf_probability["TP"] < snp_cutoff_global.min_score)) | (
~hl.is_snp(ht.alleles[0], ht.alleles[1])
& (ht.rf_probability["TP"] < indel_cutoff_global.min_score))
# Fix annotations for release
annotations_expr = {
"rf_positive_label": hl.or_else(ht.tp, False),
"rf_negative_label": ht.fail_hard_filters,
"rf_probability": ht.rf_probability["TP"],
}
ht = ht.transmute(filters=add_filters_expr(filters=filters),
**annotations_expr)
ht = ht.annotate_globals(rf_snv_cutoff=snp_cutoff_global,
rf_indel_cutoff=indel_cutoff_global)
return ht
def generate_final_rf_ht(
ht: hl.Table,
ac0_filter_expr: hl.expr.BooleanExpression,
ts_ac_filter_expr: hl.expr.BooleanExpression,
mono_allelic_fiter_expr: hl.expr.BooleanExpression,
snp_cutoff: Union[int, float],
indel_cutoff: Union[int, float],
inbreeding_coeff_cutoff: float = INBREEDING_COEFF_HARD_CUTOFF,
determine_cutoff_from_bin: bool = False,
aggregated_bin_ht: Optional[hl.Table] = None,
bin_id: Optional[hl.expr.Int32Expression] = None,
) -> hl.Table:
"""
Prepares finalized RF model given an RF result table from `rf.apply_rf_model` and cutoffs for filtering.
If `determine_cutoff_from_bin` is True, `aggregated_bin_ht` must be supplied to determine the SNP and indel RF
probabilities to use as cutoffs from an aggregated quantile bin Table like one created by
`compute_grouped_binned_ht` in combination with `score_bin_agg`.
:param ht: RF result table from `rf.apply_rf_model` to prepare as the final RF Table
:param ac0_filter_expr: Expression that indicates if a variant should be filtered as allele count 0 (AC0)
:param ts_ac_filter_expr: Expression in `ht` that indicates if a variant is a transmitted singleton
:param mono_allelic_fiter_expr: Expression indicating if a variant is mono-allelic
:param snp_cutoff: RF probability or bin (if `determine_cutoff_from_bin` True) to use for SNP variant QC filter
:param indel_cutoff: RF probability or bin (if `determine_cutoff_from_bin` True) to use for indel variant QC filter
:param inbreeding_coeff_cutoff: InbreedingCoeff hard filter to use for variants
:param determine_cutoff_from_bin: If True RF probability will be determined using bin info in `aggregated_bin_ht`
:param aggregated_bin_ht: File with aggregate counts of variants based on quantile bins
:param bin_id: Name of bin to use in 'bin_id' column of `aggregated_bin_ht` to use to determine probability cutoff
:return: Finalized random forest Table annotated with variant filters
"""
# Determine SNP and indel RF cutoffs if given bin instead of RF probability
if determine_cutoff_from_bin:
snp_rf_cutoff, indel_rf_cutoff = aggregated_bin_ht.aggregate([
hl.agg.filter(
snv
& (aggregated_bin_ht.bin_id == bin_id)
& (aggregated_bin_ht.bin == cutoff),
hl.agg.min(aggregated_bin_ht.min_score),
) for snv, cutoff in [
(aggregated_bin_ht.snv, snp_cutoff),
(~aggregated_bin_ht.snv, indel_cutoff),
]
])
snp_cutoff_global = hl.struct(bin=snp_cutoff, min_score=snp_rf_cutoff)
indel_cutoff_global = hl.struct(bin=indel_cutoff,
min_score=indel_rf_cutoff)
logger.info(
f"Using a SNP RF probability cutoff of {snp_rf_cutoff} and an indel RF probability cutoff of {indel_rf_cutoff}."
)
else:
snp_cutoff_global = hl.struct(min_score=snp_cutoff)
indel_cutoff_global = hl.struct(min_score=indel_cutoff)
# Add filters to RF HT
filters = dict()
if ht.any(hl.is_missing(ht.rf_probability["TP"])):
raise ValueError("Missing RF probability!")
filters["RF"] = (
hl.is_snp(ht.alleles[0], ht.alleles[1])
& (ht.rf_probability["TP"] < snp_cutoff_global.min_score)) | (
~hl.is_snp(ht.alleles[0], ht.alleles[1])
& (ht.rf_probability["TP"] < indel_cutoff_global.min_score))
filters["InbreedingCoeff"] = hl.or_else(
ht.InbreedingCoeff < inbreeding_coeff_cutoff, False)
filters["AC0"] = ac0_filter_expr
filters[
"MonoAllelic"] = mono_allelic_fiter_expr # TODO: Do others agree that we should add this to gnomAD like we did for UKBB?
# Fix annotations for release
annotations_expr = {
"rf_positive_label":
hl.or_else(ht.tp, False),
"rf_negative_label":
ht.fail_hard_filters,
"transmitted_singleton":
hl.or_missing(ts_ac_filter_expr, ht.transmitted_singleton),
"rf_probability":
ht.rf_probability["TP"],
}
if "feature_imputed" in ht.row:
annotations_expr.update({
x: hl.or_missing(~ht.feature_imputed[x], ht[x])
for x in [f for f in ht.row.feature_imputed]
})
ht = ht.transmute(filters=add_filters_expr(filters=filters),
**annotations_expr)
ht = ht.annotate_globals(rf_snv_cutoff=snp_cutoff_global,
rf_indel_cutoff=indel_cutoff_global)
return ht
def generate_final_filter_ht(
ht: hl.Table,
model_name: str,
score_name: str,
ac0_filter_expr: hl.expr.BooleanExpression,
ts_ac_filter_expr: hl.expr.BooleanExpression,
mono_allelic_flag_expr: hl.expr.BooleanExpression,
inbreeding_coeff_cutoff: float = INBREEDING_COEFF_HARD_CUTOFF,
snp_bin_cutoff: int = None,
indel_bin_cutoff: int = None,
snp_score_cutoff: float = None,
indel_score_cutoff: float = None,
aggregated_bin_ht: Optional[hl.Table] = None,
bin_id: Optional[str] = None,
vqsr_ht: hl.Table = None,
) -> hl.Table:
"""
Prepares finalized filtering model given a filtering HT from `rf.apply_rf_model` or VQSR and cutoffs for filtering.
.. note::
- `snp_bin_cutoff` and `snp_score_cutoff` are mutually exclusive, and one must be supplied.
- `indel_bin_cutoff` and `indel_score_cutoff` are mutually exclusive, and one must be supplied.
- If a `snp_bin_cutoff` or `indel_bin_cutoff` cutoff is supplied then an `aggregated_bin_ht` and `bin_id` must
also be supplied to determine the SNP and indel scores to use as cutoffs from an aggregated bin Table like
one created by `compute_grouped_binned_ht` in combination with `score_bin_agg`.
:param ht: Filtering Table from `rf.apply_rf_model` or VQSR to prepare as the final filter Table
:param model_name: Filtering model name to use in the 'filters' field (VQSR or RF)
:param score_name: Name to use for the filtering score annotation. This will be used in place of 'score' in the
release HT info struct and the INFO field of the VCF (e.g. RF or AS_VQSLOD)
:param ac0_filter_expr: Expression that indicates if a variant should be filtered as allele count 0 (AC0)
:param ts_ac_filter_expr: Allele count expression in `ht` to use as a filter for determining a transmitted singleton
:param mono_allelic_flag_expr: Expression indicating if a variant is mono-allelic
:param inbreeding_coeff_cutoff: InbreedingCoeff hard filter to use for variants
:param snp_bin_cutoff: Bin cutoff to use for SNP variant QC filter. Can't be used with `snp_score_cutoff`
:param indel_bin_cutoff: Bin cutoff to use for indel variant QC filter. Can't be used with `indel_score_cutoff`
:param snp_score_cutoff: Score cutoff (e.g. RF probability or AS_VQSLOD) to use for SNP variant QC filter. Can't be used with `snp_bin_cutoff`
:param indel_score_cutoff: Score cutoff (e.g. RF probability or AS_VQSLOD) to use for indel variant QC filter. Can't be used with `indel_bin_cutoff`
:param aggregated_bin_ht: Table with aggregate counts of variants based on bins
:param bin_id: Name of bin to use in 'bin_id' column of `aggregated_bin_ht` to use to determine probability cutoff
:param vqsr_ht: If a VQSR HT is supplied a 'vqsr' annotation containing AS_VQSLOD, AS_culprit, NEGATIVE_TRAIN_SITE,
and POSITIVE_TRAIN_SITE will be included in the returned Table
:return: Finalized random forest Table annotated with variant filters
"""
if snp_bin_cutoff is not None and snp_score_cutoff is not None:
raise ValueError(
"snp_bin_cutoff and snp_score_cutoff are mutually exclusive, please only supply one SNP filtering cutoff."
)
if indel_bin_cutoff is not None and indel_score_cutoff is not None:
raise ValueError(
"indel_bin_cutoff and indel_score_cutoff are mutually exclusive, please only supply one indel filtering cutoff."
)
if snp_bin_cutoff is None and snp_score_cutoff is None:
raise ValueError(
"One (and only one) of the parameters snp_bin_cutoff and snp_score_cutoff must be supplied."
)
if indel_bin_cutoff is None and indel_score_cutoff is None:
raise ValueError(
"One (and only one) of the parameters indel_bin_cutoff and indel_score_cutoff must be supplied."
)
if (snp_bin_cutoff is not None or indel_bin_cutoff
is not None) and (aggregated_bin_ht is None or bin_id is None):
raise ValueError(
"If using snp_bin_cutoff or indel_bin_cutoff, both aggregated_bin_ht and bin_id must be supplied"
)
# Determine SNP and indel score cutoffs if given bin instead of score
if snp_bin_cutoff:
snp_score_cutoff = aggregated_bin_ht.aggregate(
hl.agg.filter(
aggregated_bin_ht.snv
& (aggregated_bin_ht.bin_id == bin_id)
& (aggregated_bin_ht.bin == snp_bin_cutoff),
hl.agg.min(aggregated_bin_ht.min_score),
))
snp_cutoff_global = hl.struct(bin=snp_bin_cutoff,
min_score=snp_score_cutoff)
if indel_bin_cutoff:
indel_score_cutoff = aggregated_bin_ht.aggregate(
hl.agg.filter(
~aggregated_bin_ht.snv
& (aggregated_bin_ht.bin_id == bin_id)
& (aggregated_bin_ht.bin == indel_bin_cutoff),
hl.agg.min(aggregated_bin_ht.min_score),
))
indel_cutoff_global = hl.struct(bin=indel_bin_cutoff,
min_score=indel_score_cutoff)
min_score = ht.aggregate(hl.agg.min(ht.score))
max_score = ht.aggregate(hl.agg.max(ht.score))
if snp_score_cutoff:
if snp_score_cutoff < min_score or snp_score_cutoff > max_score:
raise ValueError(
"snp_score_cutoff is not within the range of score.")
snp_cutoff_global = hl.struct(min_score=snp_score_cutoff)
if indel_score_cutoff:
if indel_score_cutoff < min_score or indel_score_cutoff > max_score:
raise ValueError(
"indel_score_cutoff is not within the range of score.")
indel_cutoff_global = hl.struct(min_score=indel_score_cutoff)
logger.info(
f"Using a SNP score cutoff of {snp_score_cutoff} and an indel score cutoff of {indel_score_cutoff}."
)
# Add filters to HT
filters = dict()
if ht.any(hl.is_missing(ht.score)):
ht.filter(hl.is_missing(ht.score)).show()
raise ValueError("Missing Score!")
filters[model_name] = (hl.is_missing(ht.score)
| (hl.is_snp(ht.alleles[0], ht.alleles[1])
& (ht.score < snp_cutoff_global.min_score))
| (~hl.is_snp(ht.alleles[0], ht.alleles[1])
& (ht.score < indel_cutoff_global.min_score)))
filters["InbreedingCoeff"] = hl.or_else(
ht.InbreedingCoeff < inbreeding_coeff_cutoff, False)
filters["AC0"] = ac0_filter_expr
annotations_expr = dict()
if model_name == "RF":
# Fix annotations for release
annotations_expr = annotations_expr.update({
"positive_train_site":
hl.or_else(ht.positive_train_site, False),
"rf_tp_probability":
ht.rf_probability["TP"],
})
annotations_expr.update({
"transmitted_singleton":
hl.or_missing(ts_ac_filter_expr, ht.transmitted_singleton)
})
if "feature_imputed" in ht.row:
annotations_expr.update({
x: hl.or_missing(~ht.feature_imputed[x], ht[x])
for x in [f for f in ht.row.feature_imputed]
})
ht = ht.transmute(
filters=add_filters_expr(filters=filters),
monoallelic=mono_allelic_flag_expr,
**{score_name: ht.score},
**annotations_expr,
)
bin_names = [x for x in ht.row if x.endswith("bin")]
bin_names = [(
x,
x.split("adj_")[0] +
x.split("adj_")[1] if len(x.split("adj_")) == 2 else "raw_" + x,
) for x in bin_names]
ht = ht.transmute(**{j: ht[i] for i, j in bin_names})
ht = ht.annotate_globals(
bin_stats=hl.struct(**{j: ht.bin_stats[i]
for i, j in bin_names}),
filtering_model=hl.struct(
model_name=model_name,
score_name=score_name,
snv_cutoff=snp_cutoff_global,
indel_cutoff=indel_cutoff_global,
),
inbreeding_coeff_cutoff=inbreeding_coeff_cutoff,
)
if vqsr_ht:
vqsr = vqsr_ht[ht.key]
ht = ht.annotate(
vqsr=hl.struct(
AS_VQSLOD=vqsr.info.AS_VQSLOD,
AS_culprit=vqsr.info.AS_culprit,
NEGATIVE_TRAIN_SITE=vqsr.info.NEGATIVE_TRAIN_SITE,
POSITIVE_TRAIN_SITE=vqsr.info.POSITIVE_TRAIN_SITE,
),
SOR=vqsr.info.
SOR, # NOTE: This was required for v3.1, we now compute this in `get_site_info_expr`
)
ht = ht.drop("AS_culprit")
return ht
