def annotation_scheduler():
""" This is run on scheduling_single_worker queue to avoid race conditions """
LOCK_EXPIRE = 60 * 5 # 5 minutes
lock_id = "annotation-scheduler-lock"
# cache.add fails if if the key already exists
acquire_lock = lambda: cache.add(lock_id, "true", LOCK_EXPIRE)
release_lock = lambda: cache.delete(lock_id)
try:
if acquire_lock():
try:
logging.info("Got the lock for annotation scheduler")
for genome_build in GenomeBuild.builds_with_annotation():
annotation_version = AnnotationVersion.latest(genome_build)
variant_annotation_version = annotation_version.variant_annotation_version
while True:
range_lock = _handle_variant_annotation_version(
variant_annotation_version)
if range_lock is None:
break
finally:
logging.info("Releasing lock")
release_lock()
else:
logging.info("Someone else has %s", lock_id)
except:
log_traceback()
def annotation_versions(request):
anno_versions = {}
# Create VariantAnnotationVersion for build if not exists
for genome_build in GenomeBuild.builds_with_annotation():
try:
get_variant_annotation_version(genome_build)
except:
log_traceback()
qs = AnnotationVersion.objects.filter(
genome_build=genome_build).order_by("-annotation_date")
vep_command = get_vep_command("in.vcf", "out.vcf", genome_build,
genome_build.annotation_consortium)
vep_command = " ".join(vep_command).replace(" -", "\n")
anno_versions[genome_build.name] = (vep_command, qs)
context = {"annotation_versions": anno_versions}
return render(request, "annotation/annotation_versions.html", context)
def handle(self, *args, **options):
for genome_build in GenomeBuild.builds_with_annotation():
variant_qs = Variant.objects.filter(
Variant.get_contigs_q(genome_build), varianttag__isnull=False)
populate_clingen_alleles_for_variants(
genome_build, variant_qs) # Will add VariantAlleles
va_collection = VariantAlleleCollectionSource.objects.create(
genome_build=genome_build)
records = []
for va in VariantAllele.objects.filter(
variant__in=variant_qs): # VariantAlleles added above
records.append(
VariantAlleleCollectionRecord(collection=va_collection,
variant_allele=va))
if records:
VariantAlleleCollectionRecord.objects.bulk_create(
records, batch_size=2000)
create_liftover_pipelines(admin_bot(), va_collection,
ImportSource.COMMAND_LINE, genome_build)
def variant_annotation_runs(request):
as_display = dict(AnnotationStatus.choices)
genome_build_field_counts = {}
genome_build_summary = {}
if request.method == "POST":
for genome_build in GenomeBuild.builds_with_annotation():
annotation_runs = AnnotationRun.objects.filter(
annotation_range_lock__version__genome_build=genome_build)
message = None
if f"set-non-finished-to-error-{genome_build.name}" in request.POST:
num_errored = 0
non_finished_statuses = [
AnnotationStatus.FINISHED, AnnotationStatus.ERROR
]
for annotation_run in annotation_runs.exclude(
status__in=non_finished_statuses):
if celery_task := annotation_run.task_id:
logging.info("Terminating celery job '%s'",
celery_task)
app.control.revoke(
celery_task, terminate=True) # @UndefinedVariable
annotation_run.error_exception = "Manually failed"
annotation_run.save()
num_errored += 1
message = f"{genome_build} - set {num_errored} annotation runs to Error"
elif f"retry-annotation-runs-{genome_build.name}" in request.POST:
num_retrying = 0
for annotation_run in annotation_runs.filter(
status=AnnotationStatus.ERROR):
annotation_run_retry(annotation_run)
num_retrying += 1
message = f"{genome_build} - retrying {num_retrying} annotation runs."
if message:
messages.add_message(request, messages.INFO, message)
def annotation(request):
# Set Variables to None for uninstalled components, the template will show installation instructions
ensembl_biomart_transcript_genes = None
diagnostic_gene_list = None
build_contigs = get_build_contigs()
genome_build_annotations = {}
builds_ok = []
for genome_build in GenomeBuild.builds_with_annotation():
annotation_details = _get_build_annotation_details(
build_contigs, genome_build)
genome_build_annotations[genome_build.name] = annotation_details
builds_ok.append(annotation_details.get("ok", False))
gene_symbol_alias_counts = get_field_counts(GeneSymbolAlias.objects.all(),
"source")
if gene_symbol_alias_counts:
gene_symbol_alias_counts = {
GeneSymbolAliasSource(k).label: v
for k, v in gene_symbol_alias_counts.items()
}
all_ontologies_accounted_for = True
ontology_counts = list()
for service in [
OntologyService.MONDO, OntologyService.OMIM, OntologyService.HPO,
OntologyService.HGNC
]:
# don't report HGNC as it's just there as a stub for other items to relate to
count = OntologyTerm.objects.filter(ontology_service=service).count()
ontology_counts.append({"service": service, "count": count})
ontology_services = [
OntologyService.MONDO, OntologyService.OMIM, OntologyService.HPO,
OntologyService.HGNC
]
ontology_relationship_counts = dict()
for first_index, first_service in enumerate(ontology_services):
for second_service in ontology_services[first_index:]:
join_count = OntologyTermRelation.objects.filter(
source_term__ontology_service=first_service,
dest_term__ontology_service=second_service).count()
if first_service != second_service:
reverse_count = OntologyTermRelation.objects.filter(
source_term__ontology_service=second_service,
dest_term__ontology_service=first_service).count()
join_count += reverse_count
ontology_relationship_counts[
f"{first_service}{second_service}"] = join_count
ontology_relationship_counts[
f"{second_service}{first_service}"] = join_count
ontology_imports = list()
for context in [
"mondo_file", "gencc_file", "hpo_file", "omim_file",
"biomart_omim_aliases", "phenotype_to_genes"
]:
last_import = OntologyImport.objects.filter(
context=context).order_by('-created').first()
if not last_import and context != "omim_file": # don't complain about omim_file not being imported as not available to environments without license
all_ontologies_accounted_for = False
ontology_imports.append({
"context": context,
"last_import": last_import
})
diagnostic = GeneListCategory.objects.get(name='Diagnostic')
diagnostic_gene_list_count = diagnostic.genelist_set.all().count()
if diagnostic_gene_list_count:
diagnostic_gene_list = f"{diagnostic_gene_list_count} diagnostic gene lists"
clinvar_citations = ClinVarCitation.objects.count()
if clinvar_citations:
num_cached_clinvar_citations = CachedCitation.objects.count()
clinvar_citations = f"{clinvar_citations} ClinVar citations ({num_cached_clinvar_citations} cached)"
hpa_version = HumanProteinAtlasAnnotationVersion.objects.order_by(
"-annotation_date").first()
hpa_counts = HumanProteinAtlasAnnotation.objects.filter(
version=hpa_version).count()
somalier = None
if somalier_enabled := settings.SOMALIER.get("enabled"):
somalier = _verify_somalier_config()
annotation_run.error_exception = "Manually failed"
annotation_run.save()
num_errored += 1
message = f"{genome_build} - set {num_errored} annotation runs to Error"
elif f"retry-annotation-runs-{genome_build.name}" in request.POST:
num_retrying = 0
for annotation_run in annotation_runs.filter(
status=AnnotationStatus.ERROR):
annotation_run_retry(annotation_run)
num_retrying += 1
message = f"{genome_build} - retrying {num_retrying} annotation runs."
if message:
messages.add_message(request, messages.INFO, message)
for genome_build in GenomeBuild.builds_with_annotation():
qs = AnnotationRun.objects.filter(
annotation_range_lock__version__genome_build=genome_build)
field_counts = get_field_counts(qs, "status")
summary_data = Counter()
for field, count in field_counts.items():
summary = AnnotationStatus.get_summary_state(field)
summary_data[summary] += count
genome_build_summary[genome_build.pk] = summary_data
genome_build_field_counts[genome_build.pk] = {
as_display[k]: v
for k, v in field_counts.items()
}
context = {
"genome_build_summary": genome_build_summary,