def nirvana(dataset: Union[MatrixTable, Table], config, block_size=500000, name='nirvana'):
"""Annotate variants using `Nirvana <https://github.com/Illumina/Nirvana>`_.
.. include:: ../_templates/experimental.rst
.. include:: ../_templates/req_tvariant.rst
:func:`.nirvana` runs `Nirvana
<https://github.com/Illumina/Nirvana>`_ on the current dataset and adds a
new row field in the location specified by `name`.
Examples
--------
Add Nirvana annotations to the dataset:
>>> result = hl.nirvana(dataset, "data/nirvana.properties") # doctest: +SKIP
**Configuration**
:func:`.nirvana` requires a configuration file. The format is a
`.properties file <https://en.wikipedia.org/wiki/.properties>`__, where each
line defines a property as a key-value pair of the form ``key = value``.
:func:`.nirvana` supports the following properties:
- **hail.nirvana.dotnet** -- Location of dotnet. Optional, default: dotnet.
- **hail.nirvana.path** -- Value of the PATH environment variable when
invoking Nirvana. Optional, by default PATH is not set.
- **hail.nirvana.location** -- Location of Nirvana.dll. Required.
- **hail.nirvana.reference** -- Location of reference genome. Required.
- **hail.nirvana.cache** -- Location of cache. Required.
- **hail.nirvana.supplementaryAnnotationDirectory** -- Location of
Supplementary Database. Optional, no supplementary database by default.
Here is an example ``nirvana.properties`` configuration file:
.. code-block:: text
hail.nirvana.location = /path/to/dotnet/netcoreapp2.0/Nirvana.dll
hail.nirvana.reference = /path/to/nirvana/References/Homo_sapiens.GRCh37.Nirvana.dat
hail.nirvana.cache = /path/to/nirvana/Cache/GRCh37/Ensembl
hail.nirvana.supplementaryAnnotationDirectory = /path/to/nirvana/SupplementaryDatabase/GRCh37
**Annotations**
A new row field is added in the location specified by `name` with the
following schema:
.. code-block:: text
struct {
chromosome: str,
refAllele: str,
position: int32,
altAlleles: array<str>,
cytogeneticBand: str,
quality: float64,
filters: array<str>,
jointSomaticNormalQuality: int32,
copyNumber: int32,
strandBias: float64,
recalibratedQuality: float64,
variants: array<struct {
altAllele: str,
refAllele: str,
chromosome: str,
begin: int32,
end: int32,
phylopScore: float64,
isReferenceMinor: bool,
variantType: str,
vid: str,
hgvsg: str,
isRecomposedVariant: bool,
isDecomposedVariant: bool,
regulatoryRegions: array<struct {
id: str,
type: str,
consequence: set<str>
}>,
clinvar: array<struct {
id: str,
reviewStatus: str,
isAlleleSpecific: bool,
alleleOrigins: array<str>,
refAllele: str,
altAllele: str,
phenotypes: array<str>,
medGenIds: array<str>,
omimIds: array<str>,
orphanetIds: array<str>,
significance: str,
lastUpdatedDate: str,
pubMedIds: array<str>
}>,
cosmic: array<struct {
id: str,
isAlleleSpecific: bool,
refAllele: str,
altAllele: str,
gene: str,
sampleCount: int32,
studies: array<struct {
id: int32,
histology: str,
primarySite: str
}>
}>,
dbsnp: struct {
ids: array<str>
},
globalAllele: struct {
globalMinorAllele: str,
globalMinorAlleleFrequency: float64
},
gnomad: struct {
coverage: str,
allAf: float64,
allAc: int32,
allAn: int32,
allHc: int32,
afrAf: float64,
afrAc: int32,
afrAn: int32,
afrHc: int32,
amrAf: float64,
amrAc: int32,
amrAn: int32,
amrHc: int32,
easAf: float64,
easAc: int32,
easAn: int32,
easHc: int32,
finAf: float64,
finAc: int32,
finAn: int32,
finHc: int32,
nfeAf: float64,
nfeAc: int32,
nfeAn: int32,
nfeHc: int32,
othAf: float64,
othAc: int32,
othAn: int32,
othHc: int32,
asjAf: float64,
asjAc: int32,
asjAn: int32,
asjHc: int32,
failedFilter: bool
},
gnomadExome: struct {
coverage: str,
allAf: float64,
allAc: int32,
allAn: int32,
allHc: int32,
afrAf: float64,
afrAc: int32,
afrAn: int32,
afrHc: int32,
amrAf: float64,
amrAc: int32,
amrAn: int32,
amrHc: int32,
easAf: float64,
easAc: int32,
easAn: int32,
easHc: int32,
finAf: float64,
finAc: int32,
finAn: int32,
finHc: int32,
nfeAf: float64,
nfeAc: int32,
nfeAn: int32,
nfeHc: int32,
othAf: float64,
othAc: int32,
othAn: int32,
othHc: int32,
asjAf: float64,
asjAc: int32,
asjAn: int32,
asjHc: int32,
sasAf: float64,
sasAc: int32,
sasAn: int32,
sasHc: int32,
failedFilter: bool
},
topmed: struct {
failedFilter: bool,
allAc: int32,
allAn: int32,
allAf: float64,
allHc: int32
},
oneKg: struct {
ancestralAllele: str,
allAf: float64,
allAc: int32,
allAn: int32,
afrAf: float64,
afrAc: int32,
afrAn: int32,
amrAf: float64,
amrAc: int32,
amrAn: int32,
easAf: float64,
easAc: int32,
easAn: int32,
eurAf: float64,
eurAc: int32,
eurAn: int32,
sasAf: float64,
sasAc: int32,
sasAn: int32
},
mitomap: array<struct {
refAllele: str,
altAllele: str,
diseases : array<str>,
hasHomoplasmy: bool,
hasHeteroplasmy: bool,
status: str,
clinicalSignificance: str,
scorePercentile: float64,
isAlleleSpecific: bool,
chromosome: str,
begin: int32,
end: int32,
variantType: str
}
transcripts: struct {
refSeq: array<struct {
transcript: str,
bioType: str,
aminoAcids: str,
cdnaPos: str,
codons: str,
cdsPos: str,
exons: str,
introns: str,
geneId: str,
hgnc: str,
consequence: array<str>,
hgvsc: str,
hgvsp: str,
isCanonical: bool,
polyPhenScore: float64,
polyPhenPrediction: str,
proteinId: str,
proteinPos: str,
siftScore: float64,
siftPrediction: str
}>,
ensembl: array<struct {
transcript: str,
bioType: str,
aminoAcids: str,
cdnaPos: str,
codons: str,
cdsPos: str,
exons: str,
introns: str,
geneId: str,
hgnc: str,
consequence: array<str>,
hgvsc: str,
hgvsp: str,
isCanonical: bool,
polyPhenScore: float64,
polyPhenPrediction: str,
proteinId: str,
proteinPos: str,
siftScore: float64,
siftPrediction: str
}>
},
overlappingGenes: array<str>
}>
genes: array<struct {
name: str,
omim: array<struct {
mimNumber: int32,
hgnc: str,
description: str,
phenotypes: array<struct {
mimNumber: int32,
phenotype: str,
mapping: str,
inheritance: array<str>,
comments: str
}>
}>
exac: struct {
pLi: float64,
pRec: float64,
pNull: float64
}
}>
}
Parameters
----------
dataset : :class:`.MatrixTable` or :class:`.Table`
Dataset.
config : :obj:`str`
Path to Nirvana configuration file.
block_size : :obj:`int`
Number of rows to process per Nirvana invocation.
name : :obj:`str`
Name for resulting row field.
Returns
-------
:class:`.MatrixTable` or :class:`.Table`
Dataset with new row-indexed field `name` containing Nirvana annotations.
"""
if isinstance(dataset, MatrixTable):
require_row_key_variant(dataset, 'nirvana')
ht = dataset.select_rows().rows()
else:
require_table_key_variant(dataset, 'nirvana')
ht = dataset.select()
annotations = Table(TableToTableApply(ht._tir,
{'name': 'Nirvana',
'config': config,
'blockSize': block_size}
)).persist()
if isinstance(dataset, MatrixTable):
return dataset.annotate_rows(**{name: annotations[dataset.row_key].nirvana})
else:
return dataset.annotate(**{name: annotations[dataset.key].nirvana})
def vep(dataset: Union[Table, MatrixTable], config, block_size=1000, name='vep', csq=False):
"""Annotate variants with VEP.
.. include:: ../_templates/req_tvariant.rst
:func:`.vep` runs `Variant Effect Predictor
<http://www.ensembl.org/info/docs/tools/vep/index.html>`__ on the
current dataset and adds the result as a row field.
Examples
--------
Add VEP annotations to the dataset:
>>> result = hl.vep(dataset, "data/vep-configuration.json") # doctest: +SKIP
Notes
-----
**Configuration**
:func:`.vep` needs a configuration file to tell it how to run VEP.
The format of the configuration file is JSON, and :func:`.vep`
expects a JSON object with three fields:
- `command` (array of string) -- The VEP command line to run. The string literal `__OUTPUT_FORMAT_FLAG__` is replaced with `--json` or `--vcf` depending on `csq`.
- `env` (object) -- A map of environment variables to values to add to the environment when invoking the command. The value of each object member must be a string.
- `vep_json_schema` (string): The type of the VEP JSON schema (as produced by the VEP when invoked with the `--json` option). Note: This is the old-style 'parseable' Hail type syntax. This will change.
Here is an example configuration file for invoking VEP release 85
installed in `/vep` with the Loftee plugin:
.. code-block:: text
{
"command": [
"/vep",
"--format", "vcf",
"__OUTPUT_FORMAT_FLAG__",
"--everything",
"--allele_number",
"--no_stats",
"--cache", "--offline",
"--minimal",
"--assembly", "GRCh37",
"--plugin", "LoF,human_ancestor_fa:/root/.vep/loftee_data/human_ancestor.fa.gz,filter_position:0.05,min_intron_size:15,conservation_file:/root/.vep/loftee_data/phylocsf_gerp.sql,gerp_file:/root/.vep/loftee_data/GERP_scores.final.sorted.txt.gz",
"-o", "STDOUT"
],
"env": {
"PERL5LIB": "/vep_data/loftee"
},
"vep_json_schema": "Struct{assembly_name:String,allele_string:String,ancestral:String,colocated_variants:Array[Struct{aa_allele:String,aa_maf:Float64,afr_allele:String,afr_maf:Float64,allele_string:String,amr_allele:String,amr_maf:Float64,clin_sig:Array[String],end:Int32,eas_allele:String,eas_maf:Float64,ea_allele:String,ea_maf:Float64,eur_allele:String,eur_maf:Float64,exac_adj_allele:String,exac_adj_maf:Float64,exac_allele:String,exac_afr_allele:String,exac_afr_maf:Float64,exac_amr_allele:String,exac_amr_maf:Float64,exac_eas_allele:String,exac_eas_maf:Float64,exac_fin_allele:String,exac_fin_maf:Float64,exac_maf:Float64,exac_nfe_allele:String,exac_nfe_maf:Float64,exac_oth_allele:String,exac_oth_maf:Float64,exac_sas_allele:String,exac_sas_maf:Float64,id:String,minor_allele:String,minor_allele_freq:Float64,phenotype_or_disease:Int32,pubmed:Array[Int32],sas_allele:String,sas_maf:Float64,somatic:Int32,start:Int32,strand:Int32}],context:String,end:Int32,id:String,input:String,intergenic_consequences:Array[Struct{allele_num:Int32,consequence_terms:Array[String],impact:String,minimised:Int32,variant_allele:String}],most_severe_consequence:String,motif_feature_consequences:Array[Struct{allele_num:Int32,consequence_terms:Array[String],high_inf_pos:String,impact:String,minimised:Int32,motif_feature_id:String,motif_name:String,motif_pos:Int32,motif_score_change:Float64,strand:Int32,variant_allele:String}],regulatory_feature_consequences:Array[Struct{allele_num:Int32,biotype:String,consequence_terms:Array[String],impact:String,minimised:Int32,regulatory_feature_id:String,variant_allele:String}],seq_region_name:String,start:Int32,strand:Int32,transcript_consequences:Array[Struct{allele_num:Int32,amino_acids:String,biotype:String,canonical:Int32,ccds:String,cdna_start:Int32,cdna_end:Int32,cds_end:Int32,cds_start:Int32,codons:String,consequence_terms:Array[String],distance:Int32,domains:Array[Struct{db:String,name:String}],exon:String,gene_id:String,gene_pheno:Int32,gene_symbol:String,gene_symbol_source:String,hgnc_id:String,hgvsc:String,hgvsp:String,hgvs_offset:Int32,impact:String,intron:String,lof:String,lof_flags:String,lof_filter:String,lof_info:String,minimised:Int32,polyphen_prediction:String,polyphen_score:Float64,protein_end:Int32,protein_start:Int32,protein_id:String,sift_prediction:String,sift_score:Float64,strand:Int32,swissprot:String,transcript_id:String,trembl:String,uniparc:String,variant_allele:String}],variant_class:String}"
}
**Annotations**
A new row field is added in the location specified by `name` with type given
by the type given by the `json_vep_schema` (if `csq` is ``False``) or
:py:data:`.tstr` (if `csq` is ``True``).
If csq is ``True``, then the CSQ header string is also added as a global
field with name ``name + '_csq_header'``.
Parameters
----------
dataset : :class:`.MatrixTable` or :class:`.Table`
Dataset.
config : :obj:`str`
Path to VEP configuration file.
block_size : :obj:`int`
Number of rows to process per VEP invocation.
name : :obj:`str`
Name for resulting row field.
csq : :obj:`bool`
If ``True``, annotates with the VCF CSQ field as a :py:data:`.tstr`.
If ``False``, annotates as the `vep_json_schema`.
Returns
-------
:class:`.MatrixTable` or :class:`.Table`
Dataset with new row-indexed field `name` containing VEP annotations.
"""
if isinstance(dataset, MatrixTable):
require_row_key_variant(dataset, 'vep')
ht = dataset.select_rows().rows()
else:
require_table_key_variant(dataset, 'vep')
ht = dataset.select()
annotations = Table(TableToTableApply(ht._tir,
{'name': 'VEP',
'config': config,
'csq': csq,
'blockSize': block_size})).persist()
if csq:
dataset = dataset.annotate_globals(
**{name + '_csq_header': annotations.index_globals()['vep_csq_header']})
if isinstance(dataset, MatrixTable):
return dataset.annotate_rows(**{name: annotations[dataset.row_key].vep})
else:
return dataset.annotate(**{name: annotations[dataset.key].vep})