"gq_hist_all",
"gq_hist_alt",
"gnomad_age_hist_het",
"gnomad_age_hist_hom",
]
}
)
# Derived top level fields
ds = ds.annotate(
alt=get_expr_for_alt_allele(ds),
chrom=get_expr_for_contig(ds.locus),
pos=ds.locus.position,
ref=get_expr_for_ref_allele(ds),
variant_id=get_expr_for_variant_id(ds),
xpos=get_expr_for_xpos(ds.locus),
)
###########
# Subsets #
###########
all_subsets = ["gnomad", "controls", "non_cancer", "non_neuro", "non_topmed"]
# There is no separate non-cancer subset for genome data. All genome samples are non-cancer.
subsets = [s for s in all_subsets if f"{s}_AC_adj" in ds.row_value.dtype.fields]
fields_per_subpopulation = ["AC_adj", "AF_adj", "AN_adj", "nhomalt_adj"]
populations = ["afr", "amr", "asj", "eas", "fin", "nfe", "oth", "sas"]
"gq_hist_all",
"gq_hist_alt",
"gnomad_age_hist_het",
"gnomad_age_hist_hom",
]
}
)
# Derived top level fields
ds = ds.annotate(
alt=get_expr_for_alt_allele(ds),
chrom=get_expr_for_contig(ds),
pos=get_expr_for_start_pos(ds),
ref=get_expr_for_ref_allele(ds),
variant_id=get_expr_for_variant_id(ds),
xpos=get_expr_for_xpos(ds),
)
###########
# Subsets #
###########
all_subsets = ["gnomad", "controls", "non_cancer", "non_neuro", "non_topmed"]
# There is no separate non-cancer subset for genome data. All genome samples are non-cancer.
subsets = [s for s in all_subsets if f"{s}_AC_adj" in ds.row_value.dtype.fields]
fields_per_subpopulation = ["AC_adj", "AF_adj", "AN_adj", "nhomalt_adj"]
populations = ["afr", "amr", "asj", "eas", "fin", "nfe", "oth", "sas"]
**{
f"main_transcript_{field}": mt.main_transcript[field]
for field in mt.main_transcript.dtype.fields
},
pos=get_expr_for_start_pos(mt),
ref=get_expr_for_ref_allele(mt),
review_status=review_status_str,
transcript_consequence_terms=get_expr_for_vep_consequence_terms_set(
vep_transcript_consequences_root=mt.sortedTranscriptConsequences),
transcript_ids=get_expr_for_vep_transcript_ids_set(
vep_transcript_consequences_root=mt.sortedTranscriptConsequences),
transcript_id_to_consequence_json=
get_expr_for_vep_transcript_id_to_consequence_map(
vep_transcript_consequences_root=mt.sortedTranscriptConsequences),
variant_id=get_expr_for_variant_id(mt),
xpos=get_expr_for_xpos(mt.locus),
)
print("\n=== Summary ===")
hl.summarize_variants(mt)
# Drop key columns for export
rows = mt.rows()
rows = rows.order_by(rows.variant_id).drop("locus", "alleles")
print("\n=== Exporting to Elasticsearch ===")
es = ElasticsearchClient(args.host, args.port)
es.export_table_to_elasticsearch(
rows,
index_name=index_name,
index_type_name=args.index_type,
def import_mnv_file(path, **kwargs):
column_types = {
"AC_mnv_ex": hl.tint,
"AC_mnv_gen": hl.tint,
"AC_mnv": hl.tint,
"AC_snp1_ex": hl.tint,
"AC_snp1_gen": hl.tint,
"AC_snp1": hl.tint,
"AC_snp2_ex": hl.tint,
"AC_snp2_gen": hl.tint,
"AC_snp2": hl.tint,
"AN_snp1_ex": hl.tfloat,
"AN_snp1_gen": hl.tfloat,
"AN_snp2_ex": hl.tfloat,
"AN_snp2_gen": hl.tfloat,
"categ": hl.tstr,
"filter_snp1_ex": hl.tarray(hl.tstr),
"filter_snp1_gen": hl.tarray(hl.tstr),
"filter_snp2_ex": hl.tarray(hl.tstr),
"filter_snp2_gen": hl.tarray(hl.tstr),
"gene_id": hl.tstr,
"gene_name": hl.tstr,
"locus.contig": hl.tstr,
"locus.position": hl.tint,
"mnv_amino_acids": hl.tstr,
"mnv_codons": hl.tstr,
"mnv_consequence": hl.tstr,
"mnv_lof": hl.tstr,
"mnv": hl.tstr,
"n_homhom_ex": hl.tint,
"n_homhom_gen": hl.tint,
"n_homhom": hl.tint,
"n_indv_ex": hl.tint,
"n_indv_gen": hl.tint,
"n_indv": hl.tint,
"snp1_amino_acids": hl.tstr,
"snp1_codons": hl.tstr,
"snp1_consequence": hl.tstr,
"snp1_lof": hl.tstr,
"snp1": hl.tstr,
"snp2_amino_acids": hl.tstr,
"snp2_codons": hl.tstr,
"snp2_consequence": hl.tstr,
"snp2_lof": hl.tstr,
"snp2": hl.tstr,
"transcript_id": hl.tstr,
}
ds = hl.import_table(path,
key="mnv",
missing="",
types=column_types,
**kwargs)
ds = ds.transmute(locus=hl.locus(ds["locus.contig"], ds["locus.position"]))
ds = ds.transmute(
contig=get_expr_for_contig(ds.locus),
pos=ds.locus.position,
xpos=get_expr_for_xpos(ds.locus),
)
ds = ds.annotate(ref=ds.mnv.split("-")[2],
alt=ds.mnv.split("-")[3],
variant_id=ds.mnv)
ds = ds.annotate(snp1_copy=ds.snp1, snp2_copy=ds.snp2)
ds = ds.transmute(constituent_snvs=[
hl.bind(
lambda variant_id_parts: hl.struct(
variant_id=ds[f"{snp}_copy"],
chrom=variant_id_parts[0],
pos=hl.int(variant_id_parts[1]),
ref=variant_id_parts[2],
alt=variant_id_parts[3],
exome=hl.or_missing(
hl.is_defined(ds[f"AN_{snp}_ex"]),
hl.struct(
filters=ds[f"filter_{snp}_ex"],
ac=ds[f"AC_{snp}_ex"],
an=hl.int(ds[f"AN_{snp}_ex"]),
),
),
genome=hl.or_missing(
hl.is_defined(ds[f"AN_{snp}_gen"]),
hl.struct(
filters=ds[f"filter_{snp}_gen"],
ac=ds[f"AC_{snp}_gen"],
an=hl.int(ds[f"AN_{snp}_gen"]),
),
),
),
ds[f"{snp}_copy"].split("-"),
) for snp in ["snp1", "snp2"]
])
ds = ds.annotate(constituent_snv_ids=[ds.snp1, ds.snp2])
ds = ds.annotate(
mnv_in_exome=ds.constituent_snvs.all(lambda s: hl.is_defined(s.exome)),
mnv_in_genome=ds.constituent_snvs.all(
lambda s: hl.is_defined(s.genome)),
)
ds = ds.transmute(
n_individuals=ds.n_indv,
ac=ds.AC_mnv,
ac_hom=ds.n_homhom,
exome=hl.or_missing(
ds.mnv_in_exome,
hl.struct(n_individuals=ds.n_indv_ex,
ac=ds.AC_mnv_ex,
ac_hom=ds.n_homhom_ex),
),
genome=hl.or_missing(
ds.mnv_in_genome,
hl.struct(n_individuals=ds.n_indv_gen,
ac=ds.AC_mnv_gen,
ac_hom=ds.n_homhom_gen),
),
)
ds = ds.drop("AC_snp1", "AC_snp2")
ds = ds.transmute(consequence=hl.struct(
category=ds.categ,
gene_id=ds.gene_id,
gene_name=ds.gene_name,
transcript_id=ds.transcript_id,
consequence=ds.mnv_consequence,
codons=ds.mnv_codons,
amino_acids=ds.mnv_amino_acids,
lof=ds.mnv_lof,
snv_consequences=[
hl.struct(
variant_id=ds[f"{snp}"],
amino_acids=ds[f"{snp}_amino_acids"],
codons=ds[f"{snp}_codons"],
consequence=ds[f"{snp}_consequence"],
lof=ds[f"{snp}_lof"],
) for snp in ["snp1", "snp2"]
],
))
# Collapse table to one row per MNV, with all consequences for the MNV collected into an array
consequences = ds.group_by(
ds.mnv).aggregate(consequences=hl.agg.collect(ds.consequence))
ds = ds.drop("consequence")
ds = ds.distinct()
ds = ds.join(consequences)
# Sort consequences by severity
ds = ds.annotate(consequences=hl.sorted(
ds.consequences,
key=lambda c: CONSEQUENCE_TERM_RANK_LOOKUP.get(c.consequence),
))
ds = ds.annotate(changes_amino_acids_for_snvs=hl.literal([0, 1]).filter(
lambda idx: ds.consequences.any(lambda csq: csq.snv_consequences[
idx].amino_acids.lower() != csq.amino_acids.lower())).map(
lambda idx: ds.constituent_snv_ids[idx]))
return ds
"region_name": hl.tstr,
}
ds = hl.import_table(args.input_url, missing="", types=column_types)
###########
# Prepare #
###########
ds = ds.annotate(
start=hl.min(ds.genomic_start, ds.genomic_end),
stop=hl.max(ds.genomic_start, ds.genomic_end),
)
ds = ds.annotate(
xstart=get_expr_for_xpos(hl.locus(ds.chr, ds.start)),
xstop=get_expr_for_xpos(hl.locus(ds.chr, ds.stop)),
)
ds = ds.drop("genomic_start", "genomic_end")
ds = ds.transmute(
chrom=ds.chr, gene_name=ds.gene, transcript_id=ds.transcript.split("\.")[0]
)
ds = ds.drop("region_name")
#########
# Write #
#########
),
alt=get_expr_for_alt_allele(mt),
chrom=get_expr_for_contig(mt),
pos=get_expr_for_start_pos(mt),
ref=get_expr_for_ref_allele(mt),
original_alt_alleles=get_expr_for_variant_ids(mt.old_locus, mt.old_alleles),
sortedTranscriptConsequences=hl.bind(
lambda genes_with_lc_lof_flag, genes_with_loftee_flag_flag: mt.sortedTranscriptConsequences.map(
lambda csq: csq.annotate(
flags=hl.struct(
lc_lof=get_expr_for_consequence_lc_lof_flag(csq),
lc_lof_in_gene=genes_with_lc_lof_flag.contains(csq.gene_id),
lof_flag=get_expr_for_consequence_loftee_flag_flag(csq),
lof_flag_in_gene=genes_with_loftee_flag_flag.contains(csq.gene_id),
nc_transcript=(csq.category == "lof") & (csq.lof == ""),
)
)
),
get_expr_for_genes_with_lc_lof_flag(mt.sortedTranscriptConsequences),
get_expr_for_genes_with_loftee_flag_flag(mt.sortedTranscriptConsequences),
),
variant_id=get_expr_for_variant_id(mt),
xpos=get_expr_for_xpos(mt),
)
# Drop key columns for export
rows = mt.rows()
rows = rows.order_by(rows.variant_id).drop("locus", "alleles")
rows.write(args.output_url)
)
p = argparse.ArgumentParser()
p.add_argument("--variant-results-url", required=True)
p.add_argument("--variant-annotations-url", required=True)
p.add_argument("--output-url", required=True)
args = p.parse_args()
hl.init(log="/tmp/hail.log")
variants = hl.read_table(args.variant_annotations_url)
variants = variants.annotate(
variant_id=get_expr_for_variant_id(variants),
chrom=get_expr_for_contig(variants.locus),
pos=variants.locus.position,
xpos=get_expr_for_xpos(variants.locus),
alt=get_expr_for_alt_allele(variants),
ref=get_expr_for_ref_allele(variants),
)
variants = variants.transmute(
transcript_id=hl.delimit(variants.transcript_id, ","),
hgvsc=hl.delimit(
variants.hgvsc.keys().map(lambda k: k + ":" + variants.hgvsc[k]), ","),
hgvsp=hl.delimit(
variants.hgvsp.keys().map(lambda k: k + ":" + variants.hgvsp[k]), ","),
)
variants = variants.annotate(flags="PASS")
variants = variants.drop("v")
results = hl.read_table(args.variant_results_url)
results = results.annotate(analysis_group=results.analysis_group.lower().