def test_import_variant_annotations_grch37(self):
genome_build = GenomeBuild.get_name_or_alias('GRCh37')
vav = self.variant_annotation_versions_by_build[genome_build.name]
print("Variants: ", Variant.objects.all().count())
print("VariantAnnotation: ", VariantAnnotation.objects.all().count())
print(f"VariantAnnotationVersion: {vav}")
annotation_range_lock, _ = get_annotation_range_lock_and_unannotated_count(
vav)
annotation_range_lock.save()
annotation_run = AnnotationRun.objects.create(
annotation_range_lock=annotation_range_lock,
vcf_annotated_filename=TestAnnotationVCF.TEST_ANNOTATION_VCF_GRCH37
)
import_vcf_annotations(annotation_run, delete_temp_files=False)
# Verify a few fields
va = VariantAnnotation.objects.get(variant_id=13629762)
self.assertEqual(va.impact, PathogenicityImpact.MODERATE)
self.assertEqual(va.dbsnp_rs_id, "rs145886106")
self.assertEqual(va.predictions_num_pathogenic, 2)
self.assertEqual(va.predictions_num_benign, 3)
self.assertTrue("COSV" in va.cosmic_id)
self.assertTrue("COSM" in va.cosmic_legacy_id)
self.assertAlmostEqual(va.gnomad_af, 0.000145)
va = VariantAnnotation.objects.get(variant_id=13638004)
self.assertEqual(va.dbsnp_rs_id, "rs776172390")
self.assertEqual(va.consequence, 'stop_gained')
self.assertEqual(va.symbol, "ARHGAP11A")
self.assertEqual(va.predictions_num_pathogenic, 1)
self.assertEqual(va.predictions_num_benign, 0)
def handle(self, *args, **options):
build_name = options["genome_build"]
genome_build = GenomeBuild.get_name_or_alias(build_name)
vep_config = VEPConfig(genome_build)
vep_version = get_vep_version(genome_build,
vep_config.annotation_consortium)
print(vep_version)
def setUpClass(cls):
super().setUpClass()
cls.user = User.objects.get_or_create(username='******')[0]
try:
cls.admin_user = User.objects.create_superuser('admin_user')
except IntegrityError:
cls.admin_user = User.objects.get(username='******')
cls.grch37 = GenomeBuild.get_name_or_alias("GRCh37")
cls.grch38 = GenomeBuild.get_name_or_alias("GRCh38")
cls.annotation_version_grch37 = get_fake_annotation_version(cls.grch37)
cls.annotation_version_grch38 = get_fake_annotation_version(cls.grch38)
cls.human_protein_atlas_tissue_sample = HumanProteinAtlasTissueSample.objects.get_or_create(
name="foo")[0]
create_fake_clinvar_data(cls.annotation_version_grch37.clinvar_version)
clinvar = ClinVar.objects.filter(
version=cls.annotation_version_grch37.clinvar_version).first()
q = Variant.get_contigs_q(cls.grch37) & Variant.get_no_reference_q()
variant = Variant.objects.filter(q).first()
create_fake_variant_annotation(
variant, cls.annotation_version_grch37.variant_annotation_version)
citation = Citation.objects.filter(
citation_source=CitationSource.PUBMED).first()
pubmed_citation = f"PubMed:{citation.citation_id}"
cls.clinvar_id = clinvar.pk
cls.variant_string = str(variant)
cls.pubmed_citations = "&".join(
(str(c)
for c in Citation.objects.all().values_list("citation_id",
flat=True)[:2]))
cls.citations_ids_list = "/".join(
(str(c)
for c in Citation.objects.all().values_list("pk", flat=True)[:2]))
cls.citations_ids_list_pubmed = pubmed_citation
transcript_version = create_fake_transcript_version(cls.grch37)
cls.gene_id = transcript_version.gene_version.gene_id
cls.gene_symbol = transcript_version.gene_version.gene_symbol
def test_import_variant_annotations_grch38(self):
genome_build = GenomeBuild.get_name_or_alias('GRCh38')
vav = self.variant_annotation_versions_by_build[genome_build.name]
print("Variants: ", Variant.objects.all().count())
print("VariantAnnotation: ", VariantAnnotation.objects.all().count())
print(f"VariantAnnotationVersion: {vav}")
annotation_range_lock, _ = get_annotation_range_lock_and_unannotated_count(
vav)
annotation_range_lock.save()
annotation_run = AnnotationRun.objects.create(
annotation_range_lock=annotation_range_lock,
vcf_annotated_filename=TestAnnotationVCF.TEST_ANNOTATION_VCF_GRCH38
)
import_vcf_annotations(annotation_run, delete_temp_files=False)
# Verify a few fields
va = VariantAnnotation.objects.get(variant_id=24601)
self.assertEqual(va.impact, PathogenicityImpact.MODERATE)
self.assertEqual(va.dbsnp_rs_id, "rs145886106")
self.assertEqual(va.predictions_num_pathogenic, 2)
self.assertEqual(va.predictions_num_benign, 3)
self.assertEqual(va.cosmic_legacy_id,
"COSM7286401") # Test it has collapsed dupes
self.assertAlmostEqual(va.gnomad_af, 0.000354913)
self.assertEqual(va.gnomad_filtered,
False) # Test it converted FILTER properly to bool
va = VariantAnnotation.objects.get(variant_id=42)
self.assertEqual(va.dbsnp_rs_id, "rs776172390")
self.assertEqual(va.consequence, 'stop_gained')
self.assertEqual(va.symbol, "ARHGAP11A")
self.assertEqual(va.predictions_num_pathogenic, 1)
self.assertEqual(va.predictions_num_benign, 0)
# 2 | 20 | -5 | 20 | 0.05 | 0.17 | 0.01 | 0.04 | OR4F5
va = VariantAnnotation.objects.get(variant_id=131165)
self.assertEqual(va.spliceai_pred_dp_ag, 2)
self.assertEqual(va.spliceai_pred_dp_al, 20)
self.assertEqual(va.spliceai_pred_dp_dg, -5)
self.assertEqual(va.spliceai_pred_dp_dl, 20)
self.assertAlmostEqual(va.spliceai_pred_ds_ag, 0.05)
self.assertAlmostEqual(va.spliceai_pred_ds_al, 0.17)
self.assertAlmostEqual(va.spliceai_pred_ds_dg, 0.01)
self.assertAlmostEqual(va.spliceai_pred_ds_dl, 0.04)
self.assertEqual(va.spliceai_gene_symbol, "OR4F5")
# RPE65:G197E | 1&1&1
va = VariantAnnotation.objects.get(variant_id=131167)
self.assertEqual(va.mastermind_count_1_cdna, 1)
self.assertEqual(va.mastermind_count_2_cdna_prot, 1)
self.assertEqual(va.mastermind_count_3_aa_change, 1)
self.assertEqual(va.mastermind_mmid3, "RPE65:G197E")
def __init__(self, user, genome_build_name):
super().__init__(user)
genome_build = GenomeBuild.get_name_or_alias(genome_build_name)
queryset = self.model.objects.filter(genome_build=genome_build)
self.queryset = queryset.values(*self.get_field_names())
self.extra_config.update({
'sortname': "created",
'sortorder': "desc",
'shrinkToFit': False
})
def handle(self, *args, **options):
annotation_version_id = options.get("annotation_version")
force = options.get("force")
if annotation_version_id:
annotation_version = AnnotationVersion.objects.get(
pk=annotation_version_id)
set_annotation_version_complete(annotation_version, force=force)
else:
for genome_build in GenomeBuild.builds_with_annotation():
annotation_version = AnnotationVersion.latest(genome_build)
set_annotation_version_complete(annotation_version,
force=force)
def handle(self, *args, **options):
logging.info("Checking/Creating VariantAnnotationVersion...")
for genome_build in GenomeBuild.builds_with_annotation():
vav = get_variant_annotation_version(genome_build)
try:
# Some upgrade migrations caused partitions to be deleted
vav.create_partition()
except DatabaseError:
pass
logging.info("Scheduling annotation...")
annotation_scheduler()
def test_version_mismatch(self):
genome_build = GenomeBuild.get_name_or_alias('GRCh37')
vav = get_variant_annotation_version(
genome_build) # Will be fake due to ANNOTATION_VEP_FAKE_VERSION
annotation_range_lock, _ = get_annotation_range_lock_and_unannotated_count(
vav)
annotation_range_lock.save()
annotation_run = AnnotationRun.objects.create(
annotation_range_lock=annotation_range_lock,
vcf_annotated_filename=TestAnnotationVCF.TEST_ANNOTATION_VCF_GRCH38
)
with self.assertRaises(VEPVersionMismatchError):
import_vcf_annotations(annotation_run, delete_temp_files=False)
def handle(self, *args, **options):
build_matchers = {b: HGVSMatcher(b) for b in GenomeBuild.builds_with_annotation()}
vc: Classification
for vc in Classification.objects.filter(variant__isnull=False):
orig_allele = vc.variant.variantallele_set.filter(genome_build=vc.get_genome_build())
allele = orig_allele.allele
transcript = vc.transcript
if transcript:
try:
matcher = build_matchers[orig_allele.genome_build]
original_hgvs = matcher.variant_to_c_hgvs_parts(vc.variant, transcript, throw_on_issue=True).full_c_hgvs
for va in allele.variantallele_set.exclude(pk=orig_allele.pk):
matcher = build_matchers[va.genome_build]
converted_hgvs = matcher.variant_to_c_hgvs_parts(va.variant, transcript, throw_on_issue=True).full_c_hgvs
if original_hgvs != converted_hgvs:
cols = [vc.pk, vc.variant.pk, orig_allele.genome_build.name, original_hgvs, converted_hgvs]
print("\t".join((str(c) for c in cols)))
except Exception as e:
print(e)
def handle(self, *args, **options):
variant_annotation_version_id = options[
"variant_annotation_version_id"]
build_name = options["genome_build"]
qs = VariantAnnotationVersion.objects.all() # --all
if variant_annotation_version_id:
qs = qs.filter(pk=variant_annotation_version_id)
elif build_name:
genome_build = GenomeBuild.get_name_or_alias(build_name)
qs = qs.filter(genome_build=genome_build)
for variant_annotation_version in qs:
logging.info("Deleting all VariantAnnotation for %s",
variant_annotation_version)
variant_annotation_version.truncate_related_objects()
# Will cascade delete AnnotationRun then VariantAnnotation
AnnotationRangeLock.objects.filter(
version=variant_annotation_version).delete()
def handle(self, *args, **options):
test = options["test"]
build_name = options["genome_build"]
genome_build = GenomeBuild.get_name_or_alias(build_name)
if test:
print("Re-generating VCF for unit test")
vep_suffix = "vep_annotated"
base_name = f"test_{genome_build.name.lower()}"
unit_test_dir = os.path.join(settings.BASE_DIR,
"annotation/tests/test_data")
vcf_filename = os.path.join(unit_test_dir, f"{base_name}.vcf")
output_dir = unit_test_dir
else:
vep_suffix = f"vep_annotated_{genome_build.name}"
output_dir = settings.ANNOTATION_VCF_DUMP_DIR
if DO_SMALL:
base_name = "dump_small"
VEP_EXAMPLES_DIR = os.path.join(
settings.ANNOTATION_VEP_BASE_DIR, "examples")
vcf_filename = os.path.join(VEP_EXAMPLES_DIR,
f"{base_name}.vcf")
else:
base_name = "test" # "dump_small"
vcf_filename = os.path.join(settings.ANNOTATION_VCF_DUMP_DIR,
f"{base_name}.vcf")
output_filename = os.path.join(output_dir,
f"{base_name}.{vep_suffix}.vcf.gz")
return_code, std_out, std_err = run_vep(
vcf_filename, output_filename, genome_build,
genome_build.annotation_consortium)
if return_code != 0:
logging.info(std_out)
logging.error(std_err)
else:
print(f"wrote: {output_filename}")
def __init__(self, **kwargs):
user = kwargs.get("user")
super().__init__(user)
fields = self.get_field_names()
self.genome_builds = list(GenomeBuild.builds_with_annotation())
if len(self.genome_builds) == 1: # No need to show
genome_build_colmodel = self._overrides.get('genome_build', {})
genome_build_colmodel['hidden'] = True
self._overrides['genome_build'] = genome_build_colmodel
user_grid_config = UserGridConfig.get(user, self.caption)
if user_grid_config.show_group_data:
qs = Analysis.filter_for_user(user)
else:
qs = Analysis.objects.filter(user=user)
qs = qs.filter(genome_build__in=self.genome_builds)
qs = qs.filter(visible=True,
template_type__isnull=True) # Hide templates
q_last_lock = Q(analysislock=F("last_lock")) | Q(
analysislock__isnull=True)
qs = qs.annotate(last_lock=Max("analysislock__pk")).filter(q_last_lock)
self.queryset = qs.values(*fields)
self.extra_config.update({'sortname': 'modified', 'sortorder': 'desc'})
def handle(self, *args, **options):
script = __file__
add_clingen_allele = options["add_clingen_allele"]
for genome_build in GenomeBuild.builds_with_annotation():
defaults = {"git_hash": Git(settings.BASE_DIR).hash}
allele_source, _ = AllClassificationsAlleleSource.objects.get_or_create(
script=script, genome_build=genome_build, defaults=defaults)
variants_qs = allele_source.get_variants_qs()
if variants_qs.count():
print(
f"{genome_build} has variants - creating Allele/ClinGen + liftover"
)
populate_clingen_alleles_for_variants(genome_build,
variants_qs)
create_liftover_pipelines(admin_bot(), allele_source,
ImportSource.COMMAND_LINE,
genome_build)
if add_clingen_allele:
# Patch those ClinGen alleles into the variant classifications
num_added_clingen_allele = 0
clingen_allele_key_null = "evidence__%s__isnull" % SpecialEKeys.CLINGEN_ALLELE_ID
for vc in Classification.objects.filter(
variant__in=variants_qs,
**{clingen_allele_key_null: True}):
_, evidence_value, _ = get_clingen_allele_and_evidence_value_for_variant(
genome_build, vc.variant)
vc.patch_value(
{SpecialEKeys.CLINGEN_ALLELE_ID: evidence_value},
source=SubmissionSource.VARIANT_GRID)
vc.save()
num_added_clingen_allele += 1
print(
f"Added {SpecialEKeys.CLINGEN_ALLELE_ID} to {num_added_clingen_allele} classifications"
)
def setUpTestData(cls):
super().setUpTestData()
cls.variant_annotation_versions_by_build = {}
VCFS = {
"GRCh37": cls.TEST_ANNOTATION_VCF_GRCH37,
"GRCh38": cls.TEST_ANNOTATION_VCF_GRCH38,
}
for genome_build_name, vcf in VCFS.items():
genome_build = GenomeBuild.get_name_or_alias(genome_build_name)
vep_dict = get_vep_version_from_vcf(vcf)
kwargs = vep_dict_to_variant_annotation_version_kwargs(vep_dict)
kwargs["genome_build"] = genome_build
vav, created = VariantAnnotationVersion.objects.get_or_create(
**kwargs)
if not created:
print("Truncating!")
vav.truncate_related_objects()
cls.variant_annotation_versions_by_build[genome_build_name] = vav
slowly_create_loci_and_variants_for_vcf(
genome_build, vcf, get_variant_id_from_info=True)
def calculate_needed_stats(run_async=False):
""" Works out what needs to be calculated and does so """
logging.info(
"Deleting Sample Stats where import_status != SUCCESS (leftovers)")
deleted = SampleStats.objects.exclude(
import_status=ImportStatus.SUCCESS).delete()
logging.info(deleted)
logging.info("Calculating Sample Stats (run_async=%s)", run_async)
for genome_build in GenomeBuild.builds_with_annotation():
try:
annotation_version = AnnotationVersion.latest(genome_build)
except InvalidAnnotationVersionError:
logging.info(
f"Skipping calculating sample stats for incomplete annotation version for build {genome_build}"
)
continue
needs_stats = Q(samplestats__isnull=True)
needs_stats |= ~Q(
samplevariantannotationstats__variant_annotation_version=
annotation_version.variant_annotation_version)
needs_stats |= ~Q(samplegeneannotationstats__gene_annotation_version=
annotation_version.gene_annotation_version)
needs_stats |= ~Q(
sampleclinvarannotationstats__clinvar_version=annotation_version.
clinvar_version)
needs_stats_passing_filters = Q(samplestatspassingfilter__isnull=True)
needs_stats_passing_filters |= ~Q(
samplevariantannotationstats__variant_annotation_version=
annotation_version.variant_annotation_version)
needs_stats_passing_filters |= ~Q(
samplegeneannotationstats__gene_annotation_version=
annotation_version.gene_annotation_version)
needs_stats_passing_filters |= ~Q(
sampleclinvarannotationstats__clinvar_version=annotation_version.
clinvar_version)
qs_filter = needs_stats | (Q(vcf__vcffilter__isnull=False)
& needs_stats_passing_filters)
samples_qs = Sample.objects.filter(
qs_filter, vcf__genome_build=genome_build).distinct()
if run_async:
logging.info("Launching sample stats jobs asynchronously")
vcf_qs = VCF.objects.filter(sample__in=samples_qs).distinct()
logging.info(f"Build: %s Samples: %d in %d VCFs", genome_build,
samples_qs.count(), vcf_qs.count())
for vcf in vcf_qs:
task = calculate_vcf_stats.si(
vcf.pk, annotation_version.pk) # @UndefinedVariable
if run_async:
task.apply_async()
else:
result = task.apply()
if result.successful():
logging.info("Successfully calculated stats for %s", vcf)
else:
logging.error("Died for VCF %s: %s", vcf, result.result)
def handle(self, *args, **options):
for genome_build in GenomeBuild.builds_with_annotation():
for vav in VariantAnnotationVersion.objects.filter(
genome_build=genome_build):
self.fix_variant_annotation_version(vav)
def filter_queryset(self, qs: QuerySet) -> QuerySet:
if genome_build_str := self.get_query_param("genome_build"):
genome_build = GenomeBuild.get_name_or_alias(genome_build_str)
qs = qs.filter(
annotation_range_lock__version__genome_build=genome_build)
def setUpClass(cls):
super().setUpClass()
grch37 = GenomeBuild.get_name_or_alias("GRCh37")
annotation_version = get_fake_annotation_version(
grch37) # Needed in cohort_hotspot_graph
cls.user_owner = User.objects.get_or_create(username='******')[0]
cls.user_non_owner = User.objects.get_or_create(
username='******')[0]
cls.vcf = VCF.objects.create(name="test_urls_vcf",
genotype_samples=1,
genome_build=grch37,
import_status=ImportStatus.SUCCESS,
user=cls.user_owner,
date=timezone.now())
cls.sample = Sample.objects.create(name="sample1",
vcf=cls.vcf,
import_status=ImportStatus.SUCCESS)
assign_permission_to_user_and_groups(cls.user_owner, cls.vcf)
assign_permission_to_user_and_groups(cls.user_owner, cls.sample)
mother_sample = Sample.objects.create(name="mother", vcf=cls.vcf)
father_sample = Sample.objects.create(name="father", vcf=cls.vcf)
cls.cohort = Cohort.objects.create(name="test_urls_cohort",
vcf=cls.vcf,
genome_build=grch37,
import_status=ImportStatus.SUCCESS)
proband_cs = CohortSample.objects.create(
cohort=cls.cohort,
sample=cls.sample,
cohort_genotype_packed_field_index=0,
sort_order=1)
mother_cs = CohortSample.objects.create(
cohort=cls.cohort,
sample=mother_sample,
cohort_genotype_packed_field_index=1,
sort_order=2)
father_cs = CohortSample.objects.create(
cohort=cls.cohort,
sample=father_sample,
cohort_genotype_packed_field_index=2,
sort_order=3)
assign_permission_to_user_and_groups(cls.user_owner, cls.cohort)
# Cohort version has been bumped every time a cohort sample has been added
collection = CohortGenotypeCollection.objects.create(
cohort=cls.cohort,
cohort_version=cls.cohort.version,
num_samples=cls.cohort.cohortsample_set.count())
cls.trio = Trio.objects.create(name="test_urls_trio",
cohort=cls.cohort,
mother=mother_cs,
mother_affected=True,
father=father_cs,
father_affected=False,
proband=proband_cs)
vcf_filename = os.path.join(
settings.BASE_DIR,
"annotation/tests/test_data/test_grch37.vep_annotated.vcf")
slowly_create_loci_and_variants_for_vcf(grch37,
vcf_filename,
get_variant_id_from_info=True)
variant = Variant.objects.filter(Variant.get_no_reference_q()).first()
CohortGenotype.objects.create(collection=collection,
variant=variant,
ref_count=1,
het_count=1,
hom_count=1,
filters="&",
samples_zygosity="ERO",
samples_allele_depth=[42, 22, 32],
samples_allele_frequency=[100, 100, 100],
samples_read_depth=[42, 22, 32],
samples_genotype_quality=[20, 20, 20],
samples_phred_likelihood=[0, 0, 0])
# Auto cohorts don't show on list
cls.cohort2 = Cohort.objects.create(name="blah cohort",
vcf=None,
genome_build=grch37,
import_status=ImportStatus.SUCCESS)
assign_permission_to_user_and_groups(cls.user_owner, cls.cohort2)
cls.analysis = Analysis.objects.create(
genome_build=grch37,
annotation_version=annotation_version,
user=cls.user_owner)
cls.node = SampleNode.objects.create(analysis=cls.analysis,
sample=cls.sample,
count=1)
gene_list = GeneList.objects.create(name="blah", user=cls.user_owner)
cls.gene_list_node = GeneListNode.objects.create(analysis=cls.analysis)
GeneListNodeGeneList.objects.create(gene_list_node=cls.gene_list_node,
gene_list=gene_list)
cls.karyomapping_analysis = KaryomappingAnalysis.objects.create(
user=cls.user_owner, name="test karyomapping", trio=cls.trio)
grid_column_name = "variantannotation__transcript_version__gene_version__gene_symbol__symbol"
analysis_params = {"analysis_id": cls.analysis.pk}
node_version_params = {
"node_id": cls.node.pk,
"node_version": cls.node.version
}
analysis_version_and_node_version_params = {
**node_version_params, "analysis_version": cls.analysis.version,
"extra_filters": "default"
}
cls.PRIVATE_OBJECT_URL_NAMES_AND_KWARGS = [
('analysis', {
"analysis_id": cls.analysis.pk
}, 200),
# analysis templates - TODO: Need to make Template and verify it is kept private
# ('analysis_templates_list', {"analysis_template_id": cls.analysis.pk}, 200),
# Node editor
('node_view', analysis_version_and_node_version_params, 200),
('node_debug', analysis_version_and_node_version_params, 200),
# Reproduce issue serializing GeneListNode
('node_debug', {
"node_id": cls.gene_list_node.pk,
"node_version": cls.gene_list_node.version,
"analysis_version": cls.analysis.version,
"extra_filters": "default"
}, 200),
('node_doc', {
"node_id": cls.node.pk
}, 200),
('node_load', {
"node_id": cls.node.pk
}, 302),
('node_column_summary', {
**node_version_params, "analysis_version":
cls.analysis.version,
"extra_filters": "default",
"grid_column_name": grid_column_name,
"significant_figures": 2
}, 200),
('node_snp_matrix', {
**node_version_params, "conversion": SNPMatrix.TOTAL_PERCENT,
"significant_figures": 2
}, 200),
('node_data', {
"node_id": cls.node.pk
}, 200),
('analysis_node_versions', analysis_params, 200),
('analysis_editor_and_grid', analysis_params, 200),
('analysis_settings', analysis_params, 200),
('analysis_settings_details_tab', analysis_params, 200),
('analysis_settings_node_counts_tab', analysis_params, 200),
('analysis_input_samples', analysis_params, 200),
# Node data
('node_data_grid', analysis_version_and_node_version_params, 200),
('node_async_wait', analysis_version_and_node_version_params, 200),
('node_errors', analysis_version_and_node_version_params, 200),
('node_method_description', node_version_params, 200),
('view_karyomapping_analysis', {
"pk": cls.karyomapping_analysis.pk
}, 200),
]
cls.PRIVATE_GRID_LIST_URLS = [
#("vcfs_grid", {}, cls.vcf),
]