def generate_family_stats(mt: hl.MatrixTable, fam_file: str, calculate_adj: bool = False) -> Tuple[hl.Table, hl.Table]:
"""
Writes bi-allelic sites MT with the following annotations:
- family_stats (TDT, Mendel Errors, AC_unrelated_qc)
- truth_data (presence in Omni, HapMap, 1KG high conf SNVs, Mills)
:param MatrixTable mt: Full MT
:param str fam_file: Fam pedigree file location
:param bool calculate_adj: Whether to also calculate family metrics for adj genotypes
:return: Table with qc annotations
:rtype: Table
"""
#mt = mt.select_cols(high_quality=mt.meta.high_quality)
mt = mt.select_rows()
mt = annotate_unrelated_sample(mt, fam_file)
# Unphased for now, since mendel_errors does not support phased alleles
mt = mt.annotate_entries(GT=unphase_call_expr(mt.GT))
ped = hl.Pedigree.read(fam_file, delimiter='\\t')
family_stats_struct, family_stats_sample_ht = family_stats(mt, ped, 'raw')
mt = mt.annotate_rows(family_stats=[family_stats_struct])
if calculate_adj:
mt = filter_to_adj(mt)
adj_family_stats_struct, adj_family_stats_sample_ht = family_stats(
mt, ped, 'adj')
family_stats_sample_ht = family_stats_sample_ht.annotate(
adj=adj_family_stats_sample_ht[family_stats_sample_ht.s])
mt = mt.annotate_rows(
family_stats=mt.family_stats.append(adj_family_stats_struct))
return mt.rows(), family_stats_sample_ht
def get_doubleton_sites(
vds_path: str = VDS_PATH,
temp_path: str = TEMP_PATH,
tranche_data: Tuple[str, int] = TRANCHE_DATA,
sparse_entries: List[str] = SPARSE_ENTRIES,
) -> hl.Table:
"""
Filter UKB VDS to bi-allelic, autosomal sites in interval QC pass regions with an adj allele count of two and no homozygotes.
:param vds_path: Path to UKB 455k VDS. Default is VDS_PATH.
:param temp_path: Path to bucket to store Table and other temporary data. Default is TEMP_PATH.
:param tranche_data: UKB tranche data (data source and data freeze number). Default is TRANCHE_DATA.
:param sparse_entries: List of fields to select from VDS. Default is SPARSE_ENTRIES.
:return: Table of high quality sites with doubletons.
"""
logger.info("Reading in VDS and filtering to bi-allelic SNPs...")
mt = hl.vds.read_vds(vds_path).variant_data
# Drop unnecessary annotations
mt = mt.select_rows().select_entries(*sparse_entries)
mt = mt.filter_rows(
bi_allelic_expr(mt) & hl.is_snp(mt.alleles[0], mt.alleles[1]))
logger.info("Filter to autosomes and splitting multiallelics...")
mt = mt.filter_rows(mt.locus.in_autosome())
# NOTE: UKB dataset does not have errors with changed loci
# (`filter_changed_loci = False` will not throw errors here)
mt = hl.experimental.sparse_split_multi(mt)
logger.info("Removing AS_lowqual sites...")
info_ht = hl.read_table(info_ht_path(*tranche_data, split=True))
mt = mt.filter_rows(~info_ht[mt.row_key].AS_lowqual)
logger.info("Filtering to interval QC pass regions...")
interval_ht = hl.read_table(interval_qc_path(*tranche_data, "autosomes"))
mt = mt.filter_rows(hl.is_defined(interval_ht[mt.locus]))
logger.info("Filtering to adj and calculating allele count...")
mt = filter_to_adj(mt)
mt = mt.annotate_rows(call_stats=hl.agg.call_stats(mt.GT, mt.alleles))
# Get AC at allele index 1 (call_stats includes a count for each allele, including reference)
mt = mt.transmute_rows(ac=mt.call_stats.AC[1],
n_hom=mt.call_stats.homozygote_count[1])
logger.info("Filtering to an allele count of two and returning...")
ht = mt.rows()
ht = ht.filter((ht.ac == 2) & (ht.n_hom == 0))
ht = ht.checkpoint(f"{temp_path}/high_quality_sites.ht", overwrite=True)
return ht
def get_gnomad_data(data_type: str,
adj: bool = False,
split: bool = True,
raw: bool = False,
non_refs_only: bool = False,
hail_version: str = CURRENT_HAIL_VERSION,
meta_version: str = None,
meta_root: Optional[str] = 'meta',
full_meta: bool = False,
fam_version: str = CURRENT_FAM,
fam_root: str = None,
duplicate_mapping_root: str = None,
release_samples: bool = False,
release_annotations: bool = None) -> hl.MatrixTable:
"""
Wrapper function to get gnomAD data as VDS. By default, returns split hardcalls (with adj annotated but not filtered)
:param str data_type: One of `exomes` or `genomes`
:param bool adj: Whether the returned data should be filtered to adj genotypes
:param bool split: Whether the dataset should be split (only applies to raw=False)
:param bool raw: Whether to return the raw (10T+) data (not recommended: unsplit, and no special consideration on sex chromosomes)
:param bool non_refs_only: Whether to return the non-ref-genotype only MT (warning: no special consideration on sex chromosomes)
:param str hail_version: One of the HAIL_VERSIONs
:param str meta_version: Version of metadata (None for current)
:param str meta_root: Where to put metadata. Set to None if no metadata is desired.
:param str full_meta: Whether to add all metadata (warning: large)
:param str fam_version: Version of metadata (default to current)
:param str fam_root: Where to put the pedigree information. Set to None if no pedigree information is desired.
:param str duplicate_mapping_root: Where to put the duplicate genome/exome samples ID mapping (default is None -- do not annotate)
:param bool release_samples: When set, filters the data to release samples only
:param str release_annotations: One of the RELEASES to add variant annotations (into va), or None for no data
:return: gnomAD hardcalls dataset with chosen annotations
:rtype: MatrixTable
"""
from gnomad.utils.filtering import filter_to_adj
if raw and split:
raise DataException(
'No split raw data. Use of hardcalls is recommended.')
if non_refs_only:
mt = hl.read_matrix_table(
get_gnomad_data_path(data_type,
split=split,
non_refs_only=non_refs_only,
hail_version=hail_version))
else:
mt = hl.read_matrix_table(
get_gnomad_data_path(data_type,
hardcalls=not raw,
split=split,
hail_version=hail_version))
if adj:
mt = filter_to_adj(mt)
if meta_root:
meta_ht = get_gnomad_meta(data_type, meta_version, full_meta=full_meta)
mt = mt.annotate_cols(**{meta_root: meta_ht[mt.s]})
if duplicate_mapping_root:
dup_ht = hl.import_table(
genomes_exomes_duplicate_ids_tsv_path,
impute=True,
key='exome_id' if data_type == "exomes" else 'genome_id')
mt = mt.annotate_cols(**{duplicate_mapping_root: dup_ht[mt.s]})
if fam_root:
fam_ht = hl.import_fam(fam_path(data_type, fam_version))
mt = mt.annotate_cols(**{fam_root: fam_ht[mt.s]})
if release_samples:
mt = mt.filter_cols(mt.meta.release)
if release_annotations:
sites_ht = get_gnomad_public_data(data_type, split)
mt = mt.select_rows(**sites_ht[mt.row_key])
mt = mt.select_globals(**sites_ht.index_globals())
return mt
def _initial_filter(data_type):
"""
Get Table of CCDG variants passing desired filters.
Possible filters are:
- Autosomes only
- SNVs only
- gnomAD v3.1.2 AC filter
- CCDG high quality exome intervals
- UK Biobank high quality exome intervals
After densification of the VDS, rows are annotated with:
- ccdg_{data_type}_was_split
- ccdg_{data_type}_AC
- ccdg_{data_type}_AN
The filtered and annotated rows are returned as a Table and are also checkpointed
:param data_type: Whether data is from genomes or exomes
:return: Table of CCDG filtered variants
"""
logger.info(
"Loading CCDG %s VDS and splitting multi-allelics for initial filtering steps...",
data_type,
)
vds = get_ccdg_vds(data_type, filter_washu=filter_washu)
logger.info(
f"{vds.variant_data.count_cols()} CCDG {data_type} samples loaded..."
)
vds = hl.vds.split_multi(vds)
if autosomes_only:
logger.info("Filtering CCDG %s VDS to autosomes...", data_type)
vds = hl.vds.filter_chromosomes(vds, keep_autosomes=True)
ht = vds.variant_data.rows()
variant_filter_expr = True
if snv_only:
logger.info("Filtering CCDG %s VDS to SNVs...", data_type)
variant_filter_expr &= hl.is_snp(ht.alleles[0], ht.alleles[1])
if min_gnomad_v3_ac:
logger.info(
"Filtering CCDG %s VDS to gnomAD v3.1.2 variants with adj-filtered AC > %d...",
data_type,
min_gnomad_v3_ac,
)
variant_filter_expr &= gnomad_ht[ht.key].gnomad_AC > min_gnomad_v3_ac
vds = hl.vds.filter_variants(vds, ht.filter(variant_filter_expr), keep=True)
if high_qual_ccdg_exome_interval_only:
logger.info(
f"Filtering CCDG %s VDS to high quality (>80%% of samples with %dX coverage) CCDG exome intervals...",
data_type,
INTERVAL_DP,
)
interval_qc_ht = hl.read_table(
get_ccdg_results_path(
data_type="exomes", result=f"intervals_{INTERVAL_DP}x"
)
)
interval_qc_ht = interval_qc_ht.filter(interval_qc_ht.to_keep)
vds = hl.vds.filter_intervals(
vds, intervals=interval_qc_ht.interval.collect(), keep=True
)
if high_qual_ukbb_exome_interval_only:
if not autosomes_only:
raise ValueError(
"UK Biobank interval QC filtering is only available for autosomes!"
)
logger.info(
"Filtering CCDG %s VDS to high quality (>80%% of samples with 20X coverage) UK Biobank exome intervals...",
data_type,
)
interval_qc_ht = hl.read_table(
ukbb_interval_qc_path("broad", 7, "autosomes")
) # Note: freeze 7 is all included in gnomAD v4
interval_qc_ht = interval_qc_ht.filter(
interval_qc_ht["pct_samples_20x"] > pct_samples_ukbb_exome_interval
)
vds = hl.vds.filter_intervals(
vds, intervals=interval_qc_ht.interval.collect(), keep=True
)
logger.info("Densifying filtered CCDG %s VDS...", data_type)
mt = hl.vds.to_dense_mt(vds)
if adj_only:
mt = filter_to_adj(mt)
annotation_expr = {
f"ccdg_{data_type}_was_split": mt.was_split,
f"ccdg_{data_type}_AC": hl.agg.sum(mt.GT.n_alt_alleles()),
f"ccdg_{data_type}_AN": hl.agg.count_where(hl.is_defined(mt.GT)) * 2,
}
if min_inbreeding_coeff_threshold is not None:
annotation_expr[
f"ccdg_{data_type}_site_inbreeding_coeff"
] = bi_allelic_site_inbreeding_expr(mt.GT)
if min_hardy_weinberg_threshold is not None:
annotation_expr[f"ccdg_{data_type}_hwe"] = hl.agg.hardy_weinberg_test(
mt.GT
)
mt = mt.annotate_rows(**annotation_expr)
ht = mt.rows().checkpoint(
get_ccdg_results_path(
data_type=data_type,
mt=False,
result=f"pre_filtered_variants_interval{INTERVAL_DP}x{flag}",
),
overwrite=(not read_per_dataset_checkpoint_if_exists),
_read_if_exists=read_per_dataset_checkpoint_if_exists,
)
return ht
def main(args):
hl.init(log="/select_samples", default_reference="GRCh38")
meta_ht = hl.read_table(args.sample_metadata_ht)
meta_ht = meta_ht.filter(meta_ht.release
& hl.is_defined(meta_ht.project_meta.cram_path))
meta_ht = meta_ht.select(
cram_path=meta_ht.project_meta.cram_path,
crai_path=meta_ht.project_meta.cram_path.replace(
".cram", ".cram.crai"),
sex=meta_ht.project_meta.sex,
)
mt = MatrixTableResource(args.gnomad_mt).mt()
mt = hl.MatrixTable(
hl.ir.MatrixKeyRowsBy(mt._mir, ['locus', 'alleles'], is_sorted=True))
if args.test:
logger.info("Filtering to chrX PAR1 boundary: chrX:2781477-2781900")
mt = hl.filter_intervals(
mt, [hl.parse_locus_interval("chrX:2781477-2781900")])
meta_join = meta_ht[mt.s]
mt = mt.annotate_cols(meta=hl.struct(
sex=meta_join.sex,
cram=meta_join.cram_path,
crai=meta_join.crai_path,
))
logger.info("Filtering to releasable samples with a defined cram path")
mt = mt.filter_cols(mt.meta.release & hl.is_defined(mt.meta.cram))
mt = hl.experimental.sparse_split_multi(mt, filter_changed_loci=True)
logger.info("Adjusting samples' sex ploidy")
mt = mt.annotate_entries(GT=adjusted_sex_ploidy_expr(
mt.locus,
mt.GT,
mt.meta.sex,
xy_karyotype_str="male",
xx_karyotype_str="female",
))
mt = mt.select_entries("GT", "GQ", "DP", "AD")
logger.info(
"Filtering to entries meeting GQ, DP and other 'adj' thresholds")
mt = filter_to_adj(mt)
mt = mt.filter_rows(hl.agg.any(mt.GT.is_non_ref()))
mt = mt.filter_rows(hl.len(mt.alleles) > 1)
logger.info(
f"Taking up to {args.num_samples} samples per site where samples are het, hom_var, or hemi"
)
def sample_ordering_expr(mt):
"""It can be problematic for downstream steps when several samples have many times more variants selected
than in other samples. To avoid this, and distribute variants more evenly across samples,
add a random number as the secondary sort order. This way, when many samples have an identically high GQ
(as often happens for common variants), the same few samples don't get selected repeatedly for all common
variants.
"""
return -mt.GQ, hl.rand_unif(0, 1, seed=1)
mt = mt.annotate_rows(
samples_w_het_var=hl.agg.filter(
het_expr(mt),
hl.agg.take(het_hom_hemi_take_expr(mt),
args.num_samples,
ordering=sample_ordering_expr(mt)),
),
samples_w_hom_var=hl.agg.filter(
hom_expr(mt),
hl.agg.take(het_hom_hemi_take_expr(mt),
args.num_samples,
ordering=sample_ordering_expr(mt)),
),
samples_w_hemi_var=hl.agg.filter(
hemi_expr(mt),
hl.agg.take(het_hom_hemi_take_expr(mt),
args.num_samples,
ordering=sample_ordering_expr(mt)),
),
)
ht = mt.rows()
ht = ht.select(ht.samples_w_het_var, ht.samples_w_hom_var,
ht.samples_w_hemi_var)
ht.write(args.output_ht_path, overwrite=args.overwrite)
def get_qc_mt(
mt: hl.MatrixTable,
adj_only: bool = True,
min_af: Optional[float] = 0.001,
min_callrate: Optional[float] = 0.99,
min_inbreeding_coeff_threshold: Optional[float] = -0.8,
min_hardy_weinberg_threshold: Optional[float] = 1e-8,
apply_hard_filters: bool = True,
ld_r2: Optional[float] = 0.1,
filter_lcr: bool = True,
filter_decoy: bool = True,
filter_segdup: bool = True,
filter_exome_low_coverage_regions: bool = False,
high_conf_regions: Optional[List[str]] = None,
) -> hl.MatrixTable:
"""
Creates a QC-ready MT by keeping:
- Variants outside known problematic regions
- Bi-allelic SNVs only
- Variants passing hard thresholds
- Variants passing the set call rate and MAF thresholds
- Genotypes passing on gnomAD ADJ criteria (GQ>=20, DP>=10, AB>0.2 for hets)
In addition, the MT will be LD-pruned if `ld_r2` is set.
:param mt: Input MT
:param adj_only: If set, only ADJ genotypes are kept. This filter is applied before the call rate and AF calculation.
:param min_af: Minimum allele frequency to keep. Not applied if set to ``None``.
:param min_callrate: Minimum call rate to keep. Not applied if set to ``None``.
:param min_inbreeding_coeff_threshold: Minimum site inbreeding coefficient to keep. Not applied if set to ``None``.
:param min_hardy_weinberg_threshold: Minimum site HW test p-value to keep. Not applied if set to ``None``.
:param apply_hard_filters: Whether to apply standard GAKT default site hard filters: QD >= 2, FS <= 60 and MQ >= 30
:param ld_r2: Minimum r2 to keep when LD-pruning (set to `None` for no LD pruning)
:param filter_lcr: Filter LCR regions
:param filter_decoy: Filter decoy regions
:param filter_segdup: Filter segmental duplication regions
:param filter_exome_low_coverage_regions: If set, only high coverage exome regions (computed from gnomAD are kept)
:param high_conf_regions: If given, the data will be filtered to only include variants in those regions
:return: Filtered MT
"""
logger.info("Creating QC MatrixTable")
if ld_r2 is not None:
logger.warning(
"The LD-prune step of this function requires non-preemptible workers only!"
)
qc_mt = filter_low_conf_regions(
mt,
filter_lcr=filter_lcr,
filter_decoy=filter_decoy,
filter_segdup=filter_segdup,
filter_exome_low_coverage_regions=filter_exome_low_coverage_regions,
high_conf_regions=high_conf_regions,
)
if adj_only:
qc_mt = filter_to_adj(
qc_mt
) # TODO: Make sure that this works fine before call rate filtering
qc_mt = filter_rows_for_qc(
qc_mt,
min_af,
min_callrate,
min_inbreeding_coeff_threshold,
min_hardy_weinberg_threshold,
apply_hard_filters,
)
if ld_r2 is not None:
qc_mt = qc_mt.persist()
unfiltered_qc_mt = qc_mt.unfilter_entries()
pruned_ht = hl.ld_prune(unfiltered_qc_mt.GT, r2=ld_r2)
qc_mt = qc_mt.filter_rows(hl.is_defined(pruned_ht[qc_mt.row_key]))
qc_mt = qc_mt.annotate_globals(qc_mt_params=hl.struct(
adj_only=adj_only,
min_af=min_af if min_af is not None else hl.null(hl.tfloat32),
min_callrate=min_callrate if min_callrate is not None else hl.
null(hl.tfloat32),
inbreeding_coeff_threshold=min_inbreeding_coeff_threshold if
min_inbreeding_coeff_threshold is not None else hl.null(hl.tfloat32),
min_hardy_weinberg_threshold=min_hardy_weinberg_threshold
if min_hardy_weinberg_threshold is not None else hl.null(hl.tfloat32),
apply_hard_filters=apply_hard_filters,
ld_r2=ld_r2 if ld_r2 is not None else hl.null(hl.tfloat32),
filter_exome_low_coverage_regions=filter_exome_low_coverage_regions,
high_conf_regions=high_conf_regions
if high_conf_regions is not None else hl.null(hl.tarray(hl.tstr)),
))
return qc_mt.annotate_cols(
sample_callrate=hl.agg.fraction(hl.is_defined(qc_mt.GT)))
