def handle(self, *args, **options):
variant_collection_id = options['variant_collection_id']
variant_collection = VariantCollection.objects.get(
pk=variant_collection_id)
logging.debug("Inserting variant_collection_id = %d",
variant_collection_id)
try:
vcf_reader = cyvcf2.VCF("/dev/stdin") # Must take a filename..
bulk_inserter = BulkVCFCountInserter(variant_collection)
for v in vcf_reader:
bulk_inserter.process_entry(v)
bulk_inserter.finish() # Any leftovers
variant_collection.count = bulk_inserter.rows_processed
variant_collection.save()
except Exception:
details = get_traceback()
logging.error(details)
try:
node = variant_collection.intersectioncache.node_version.node
node.status = NodeStatus.ERROR
errors = "Error inserting variants after bed intersection:\n"
errors += details
logging.error(errors)
node.errors = errors
node.save()
except Exception as e:
logging.error(e)
create_event(None,
name="stdin_to_variant_collection",
details=details)
def run(self, cohort_gene_counts_id, update_gene_id=None):
cohort_gene_counts = CohortGeneCounts.objects.get(
pk=cohort_gene_counts_id)
cohort_gene_counts.processing_status = ProcessingStatus.PROCESSING
cohort_gene_counts.save()
try:
gene_count_type_name = cohort_gene_counts.gene_count_type.pk
show_other_counts = self._get_show_other_counts()
gene_count_type, gene_values = get_or_create_gene_count_type_and_values(
gene_count_type_name, show_other_counts=show_other_counts)
variant_annotation_version = cohort_gene_counts.variant_annotation_version
sample_gene_value_counts = self._get_gene_value_counts(
cohort_gene_counts, gene_values, variant_annotation_version,
update_gene_id)
self._save_counts(gene_count_type, gene_values,
sample_gene_value_counts,
variant_annotation_version, update_gene_id)
cohort_gene_counts.processing_status = ProcessingStatus.SUCCESS
except:
details = get_traceback()
logging.info(details)
create_event(None,
"cohort_sample_gene_damage_counts",
details,
severity=LogLevel.ERROR)
cohort_gene_counts.processing_status = ProcessingStatus.ERROR
cohort_gene_counts.save()
def annotate_variants(annotation_run_id):
annotation_run = AnnotationRun.objects.get(pk=annotation_run_id)
logging.info("annotate_variants: %s", annotation_run)
# task_id used as Celery lock
num_modified = AnnotationRun.objects.filter(
pk=annotation_run.pk,
task_id__isnull=True).update(task_id=annotate_variants.request.id)
if num_modified != 1:
msg = f"Celery could't get task_id lock on AnnotationRun: {annotation_run.pk}"
annotation_run.celery_task_logs[annotate_variants.request.id] = msg
annotation_run.save()
raise ValueError(msg)
try:
# Reload to get updated task_id
annotation_run = AnnotationRun.objects.get(pk=annotation_run_id)
if annotation_run.variant_annotation_version.gene_annotation_release is None:
# We need this so that transcript/versions are in DB so FKs link
msg = f"{annotation_run.variant_annotation_version} missing GeneAnnotationRelease"
raise InvalidAnnotationVersionError(msg)
annotation_run.task_id = annotate_variants.request.id
annotation_run.set_task_log("start", timezone.now())
annotation_run.save()
if annotation_run.vcf_annotated_filename is None:
dump_and_annotate_variants(annotation_run)
if annotation_run.vcf_annotated_filename:
import_vcf_annotations(annotation_run)
annotation_run_complete_signal.send(
sender=os.path.basename(__file__),
variant_annotation_version=annotation_run.annotation_range_lock.
version)
except Exception as e:
tb = get_traceback()
error_message = f"{e}: {annotation_run.pipeline_stderr}"
name = 'Annotation pipeline run ' + str(annotation_run.id)
report_message(message=name,
level='error',
extra_data={
'output': error_message,
'error': tb
})
annotation_run.error_exception = tb
annotation_run.set_task_log("error_exception", tb)
annotation_run.save()
create_event(None, "AnnotationRun failed", tb, severity=LogLevel.ERROR)
raise
finally:
annotation_run.set_task_log("end", timezone.now())
annotation_run.task_id = None
annotation_run.save()
def run(self, cached_web_resource_id):
cached_web_resource = CachedWebResource.objects.get(
pk=cached_web_resource_id)
cached_web_resource.import_status = ImportStatus.IMPORTING
cached_web_resource.save()
try:
self._load_cached_web_resource(cached_web_resource)
cached_web_resource.import_status = ImportStatus.SUCCESS
except:
tb = get_traceback()
cached_web_resource.import_status = ImportStatus.ERROR
cached_web_resource.description = tb
logging.error(tb)
cached_web_resource.save()
def update_node_task(node_id, version):
try:
node = AnalysisNode.objects.get_subclass(pk=node_id, version=version)
except AnalysisNode.DoesNotExist: # @UndefinedVariable
# Node was deleted or version bumped - this task is obsolete
return
errors = None
node_errors = node.get_errors()
if not node_errors:
try:
# Even if no errors now, parent nodes can be removed on us during load causing failure
# Also will throw NodeOutOfDateException if node already bumped (before calling expensive load())
node.set_node_task_and_status(update_node_task.request.id,
NodeStatus.LOADING)
node.load()
return # load already modified status, no need to save again below
except NodeOutOfDateException:
logging.warning("Node %d/%d out of date - exiting Celery task",
node.pk, node.version)
return # Other job will handle...
except OperationalError:
status = NodeStatus.CANCELLED
except NodeConfigurationException:
status = NodeStatus.ERROR_CONFIGURATION
except NodeParentErrorsException:
status = NodeStatus.ERROR_WITH_PARENT
except Exception:
errors = get_traceback()
status = NodeStatus.ERROR
else:
status = AnalysisNode.get_status_from_errors(node_errors)
try:
logging.info("Node %d/%d status: %s (old: %s) errored: %s ", node.pk,
node.version, status, node.status, errors)
if NodeStatus.is_error(status):
shadow_color = NodeColors.ERROR
else:
shadow_color = None
node.update(status=status, errors=errors, shadow_color=shadow_color)
except (IntegrityError, NodeOutOfDateException) as e:
logging.warning("Node %d/%d out of date: {%s} - exiting Celery task",
node.pk, node.version, e)
pass # Out of date or deleted - just ignore
def calculate_vcf_stats(vcf_id, annotation_version_id):
vcf = VCF.objects.get(pk=vcf_id)
try:
annotation_version = AnnotationVersion.objects.get(
pk=annotation_version_id)
msg = f"Build for annotation: {annotation_version.genome_build} and VCF: {vcf.genome_build} match"
assert annotation_version.genome_build == vcf.genome_build, msg
_actually_calculate_vcf_stats(vcf, annotation_version)
vcf.sample_set.update(import_status=ImportStatus.SUCCESS)
except:
tb = get_traceback()
create_event(None,
'calculate_vcf_stats',
details=tb,
severity=LogLevel.ERROR)
vcf.sample_set.update(import_status=ImportStatus.ERROR)
raise