def annotation_scheduler(): """ This is run on scheduling_single_worker queue to avoid race conditions """ LOCK_EXPIRE = 60 * 5 # 5 minutes lock_id = "annotation-scheduler-lock" # cache.add fails if if the key already exists acquire_lock = lambda: cache.add(lock_id, "true", LOCK_EXPIRE) release_lock = lambda: cache.delete(lock_id) try: if acquire_lock(): try: logging.info("Got the lock for annotation scheduler") for genome_build in GenomeBuild.builds_with_annotation(): annotation_version = AnnotationVersion.latest(genome_build) variant_annotation_version = annotation_version.variant_annotation_version while True: range_lock = _handle_variant_annotation_version( variant_annotation_version) if range_lock is None: break finally: logging.info("Releasing lock") release_lock() else: logging.info("Someone else has %s", lock_id) except: log_traceback()
def annotation_versions(request): anno_versions = {} # Create VariantAnnotationVersion for build if not exists for genome_build in GenomeBuild.builds_with_annotation(): try: get_variant_annotation_version(genome_build) except: log_traceback() qs = AnnotationVersion.objects.filter( genome_build=genome_build).order_by("-annotation_date") vep_command = get_vep_command("in.vcf", "out.vcf", genome_build, genome_build.annotation_consortium) vep_command = " ".join(vep_command).replace(" -", "\n") anno_versions[genome_build.name] = (vep_command, qs) context = {"annotation_versions": anno_versions} return render(request, "annotation/annotation_versions.html", context)
def handle(self, *args, **options): for genome_build in GenomeBuild.builds_with_annotation(): variant_qs = Variant.objects.filter( Variant.get_contigs_q(genome_build), varianttag__isnull=False) populate_clingen_alleles_for_variants( genome_build, variant_qs) # Will add VariantAlleles va_collection = VariantAlleleCollectionSource.objects.create( genome_build=genome_build) records = [] for va in VariantAllele.objects.filter( variant__in=variant_qs): # VariantAlleles added above records.append( VariantAlleleCollectionRecord(collection=va_collection, variant_allele=va)) if records: VariantAlleleCollectionRecord.objects.bulk_create( records, batch_size=2000) create_liftover_pipelines(admin_bot(), va_collection, ImportSource.COMMAND_LINE, genome_build)
def variant_annotation_runs(request): as_display = dict(AnnotationStatus.choices) genome_build_field_counts = {} genome_build_summary = {} if request.method == "POST": for genome_build in GenomeBuild.builds_with_annotation(): annotation_runs = AnnotationRun.objects.filter( annotation_range_lock__version__genome_build=genome_build) message = None if f"set-non-finished-to-error-{genome_build.name}" in request.POST: num_errored = 0 non_finished_statuses = [ AnnotationStatus.FINISHED, AnnotationStatus.ERROR ] for annotation_run in annotation_runs.exclude( status__in=non_finished_statuses): if celery_task := annotation_run.task_id: logging.info("Terminating celery job '%s'", celery_task) app.control.revoke( celery_task, terminate=True) # @UndefinedVariable annotation_run.error_exception = "Manually failed" annotation_run.save() num_errored += 1 message = f"{genome_build} - set {num_errored} annotation runs to Error" elif f"retry-annotation-runs-{genome_build.name}" in request.POST: num_retrying = 0 for annotation_run in annotation_runs.filter( status=AnnotationStatus.ERROR): annotation_run_retry(annotation_run) num_retrying += 1 message = f"{genome_build} - retrying {num_retrying} annotation runs." if message: messages.add_message(request, messages.INFO, message)
def annotation(request): # Set Variables to None for uninstalled components, the template will show installation instructions ensembl_biomart_transcript_genes = None diagnostic_gene_list = None build_contigs = get_build_contigs() genome_build_annotations = {} builds_ok = [] for genome_build in GenomeBuild.builds_with_annotation(): annotation_details = _get_build_annotation_details( build_contigs, genome_build) genome_build_annotations[genome_build.name] = annotation_details builds_ok.append(annotation_details.get("ok", False)) gene_symbol_alias_counts = get_field_counts(GeneSymbolAlias.objects.all(), "source") if gene_symbol_alias_counts: gene_symbol_alias_counts = { GeneSymbolAliasSource(k).label: v for k, v in gene_symbol_alias_counts.items() } all_ontologies_accounted_for = True ontology_counts = list() for service in [ OntologyService.MONDO, OntologyService.OMIM, OntologyService.HPO, OntologyService.HGNC ]: # don't report HGNC as it's just there as a stub for other items to relate to count = OntologyTerm.objects.filter(ontology_service=service).count() ontology_counts.append({"service": service, "count": count}) ontology_services = [ OntologyService.MONDO, OntologyService.OMIM, OntologyService.HPO, OntologyService.HGNC ] ontology_relationship_counts = dict() for first_index, first_service in enumerate(ontology_services): for second_service in ontology_services[first_index:]: join_count = OntologyTermRelation.objects.filter( source_term__ontology_service=first_service, dest_term__ontology_service=second_service).count() if first_service != second_service: reverse_count = OntologyTermRelation.objects.filter( source_term__ontology_service=second_service, dest_term__ontology_service=first_service).count() join_count += reverse_count ontology_relationship_counts[ f"{first_service}{second_service}"] = join_count ontology_relationship_counts[ f"{second_service}{first_service}"] = join_count ontology_imports = list() for context in [ "mondo_file", "gencc_file", "hpo_file", "omim_file", "biomart_omim_aliases", "phenotype_to_genes" ]: last_import = OntologyImport.objects.filter( context=context).order_by('-created').first() if not last_import and context != "omim_file": # don't complain about omim_file not being imported as not available to environments without license all_ontologies_accounted_for = False ontology_imports.append({ "context": context, "last_import": last_import }) diagnostic = GeneListCategory.objects.get(name='Diagnostic') diagnostic_gene_list_count = diagnostic.genelist_set.all().count() if diagnostic_gene_list_count: diagnostic_gene_list = f"{diagnostic_gene_list_count} diagnostic gene lists" clinvar_citations = ClinVarCitation.objects.count() if clinvar_citations: num_cached_clinvar_citations = CachedCitation.objects.count() clinvar_citations = f"{clinvar_citations} ClinVar citations ({num_cached_clinvar_citations} cached)" hpa_version = HumanProteinAtlasAnnotationVersion.objects.order_by( "-annotation_date").first() hpa_counts = HumanProteinAtlasAnnotation.objects.filter( version=hpa_version).count() somalier = None if somalier_enabled := settings.SOMALIER.get("enabled"): somalier = _verify_somalier_config()
annotation_run.error_exception = "Manually failed" annotation_run.save() num_errored += 1 message = f"{genome_build} - set {num_errored} annotation runs to Error" elif f"retry-annotation-runs-{genome_build.name}" in request.POST: num_retrying = 0 for annotation_run in annotation_runs.filter( status=AnnotationStatus.ERROR): annotation_run_retry(annotation_run) num_retrying += 1 message = f"{genome_build} - retrying {num_retrying} annotation runs." if message: messages.add_message(request, messages.INFO, message) for genome_build in GenomeBuild.builds_with_annotation(): qs = AnnotationRun.objects.filter( annotation_range_lock__version__genome_build=genome_build) field_counts = get_field_counts(qs, "status") summary_data = Counter() for field, count in field_counts.items(): summary = AnnotationStatus.get_summary_state(field) summary_data[summary] += count genome_build_summary[genome_build.pk] = summary_data genome_build_field_counts[genome_build.pk] = { as_display[k]: v for k, v in field_counts.items() } context = { "genome_build_summary": genome_build_summary,