def parse(self):
from assay.models import LagSediaResultRow
rows_inserted = 0
rows_failed = 0
for row_num in range(self.num_rows):
try:
if row_num >= 1:
self.header = [x.strip() for x in self.header]
row_dict = dict(zip(self.header, self.file_rows[row_num]))
lag_row = LagSediaResultRow.objects.create(
specimen_label=row_dict['specimen_label'],
assay=row_dict['assay'],
laboratory=row_dict['laboratory'],
test_date=row_dict['test_date'],
operator=row_dict['operator'],
assay_kit_lot=row_dict['assay_kit_lot'],
plate_identifier=row_dict['plate_identifier'],
test_mode=row_dict['test_mode'],
well=row_dict['well'],
specimen_purpose=row_dict['specimen_purpose'],
OD=row_dict['OD'],
calibrator_OD=row_dict['calibrator_OD'],
ODn=row_dict['ODn'],
state='pending',
fileinfo=self.upload_file)
rows_inserted += 1
except Exception, e:
self.upload_file.message = "row " + str(
row_num) + ": " + log_exception(e, logger)
self.upload_file.save()
return 0, 1
def parse(self):
from assay.models import CustomAssayResultRow
rows_inserted = 0
rows_failed = 0
for row_num in range(self.num_rows):
try:
if row_num >= 1:
self.header = [ x.strip() for x in self.header ]
row_dict = dict(zip(self.header, self.file_rows[row_num]))
custom_assay_result_row = CustomAssayResultRow.objects.create(
specimen_label = row_dict.get('specimen_label'),
assay = row_dict.get('assay'),
test_date = row_dict.get('test_date'),
operator = row_dict.get('operator'),
assay_kit_lot = row_dict.get('assay_kit_lot'),
plate_identifier = row_dict.get('plate_identifier'),
test_mode = row_dict.get('test_mode'),
specimen_purpose=row_dict.get('specimen_purpose'),
result_classification=row_dict.get('result_classification'),
result_quantitative=row_dict.get('result_quantitative'),
state='pending',
fileinfo=self.upload_file
)
rows_inserted += 1
except Exception, e:
raise
self.upload_file.message = "row " + str(row_num) + ": " + log_exception(e, logger)
self.upload_file.save()
return 0, 1
def parse(self):
from assay.models import ISGlobalResultRow
rows_inserted = 0
rows_failed = 0
for row_num in range(self.num_rows):
try:
if row_num >= 1:
self.header = [x.strip() for x in self.header]
row_dict = dict(zip(self.header, self.file_rows[row_num]))
isg_result_row = ISGlobalResultRow.objects.create(
specimen_label=row_dict['specimen_label'],
specimen_purpose=row_dict['specimen_purpose'],
classification_weighted_model=row_dict[
'classification_weighted_model'],
classification_unweighted_model=row_dict[
'classification_unweighted_model'],
state='pending',
fileinfo=self.upload_file,
assay=self.upload_file.assay,
laboratory=None)
rows_inserted += 1
except Exception, e:
raise
self.upload_file.message = "row " + str(
row_num) + ": " + log_exception(e, logger)
self.upload_file.save()
return 0, 1
def parse(self):
from assay.models import GeeniusResultRow
rows_inserted = 0
rows_failed = 0
valid_columns = set(self.registered_columns)
for row_num in range(self.num_rows):
try:
if row_num >= 1:
self.header = [x.strip() for x in self.header]
row_dict = dict(
(k, v)
for (k, v) in zip(self.header, self.file_rows[row_num])
if k in valid_columns)
GeeniusResultRow.objects.create(state='pending',
fileinfo=self.upload_file,
**row_dict)
rows_inserted += 1
except Exception, e:
self.upload_file.message = "row " + str(
row_num) + ": " + log_exception(e, logger)
self.upload_file.save()
return 0, 1
def parse(self):
from assay.models import IDEV3ResultRow
rows_inserted = 0
rows_failed = 0
for row_num in range(self.num_rows):
try:
if row_num >= 1:
self.header = [x.strip() for x in self.header]
row_dict = dict(zip(self.header, self.file_rows[row_num]))
ide_result_row = IDEV3ResultRow.objects.create(
specimen_label=row_dict['specimen_label'],
assay=row_dict['assay'],
laboratory=row_dict['laboratory'],
test_date=row_dict['test_date'],
operator=row_dict['operator'],
assay_kit_lot=row_dict['assay_kit_lot'],
plate_identifier=row_dict['plate_identifier'],
specimen_purpose=row_dict['specimen_purpose'],
exclusion=row_dict['exclusion'],
interpretation=row_dict['interpretation'],
test_mode=row_dict['test_mode'],
well_tm=row_dict['well_tm'],
well_v3=row_dict['well_v3'],
tm_OD=row_dict['tm_OD'],
v3_OD=row_dict['v3_OD'],
tm_ratio_reported=row_dict['tm_ratio_reported'],
v3_ratio_reported=row_dict['v3_ratio_reported'],
tm_ratio=row_dict['tm_ratio'],
v3_ratio=row_dict['v3_ratio'],
intermediaire_reported=row_dict[
'intermediaire_reported'],
intermediaire=row_dict['intermediaire'],
conclusion_reported=row_dict['conclusion_reported'],
conclusion=row_dict['conclusion'],
state='pending',
fileinfo=self.upload_file)
rows_inserted += 1
except Exception, e:
self.upload_file.message = "row " + str(
row_num) + ": " + log_exception(e, logger)
self.upload_file.save()
return 0, 1
def validate(self, panel_id):
from cephia.models import Specimen, Panel, Assay
from assay.models import ArchitectUnmodifiedResultRow, ArchitectUnmodifiedResult, PanelMembership
rows_validated = 0
rows_failed = 0
for architect_result_row in ArchitectUnmodifiedResultRow.objects.filter(
fileinfo=self.upload_file, state='pending'):
try:
error_msg = ''
panel = Panel.objects.get(pk=panel_id)
try:
specimen = Specimen.objects.get(
specimen_label=architect_result_row.specimen_label,
specimen_type=panel.specimen_type,
parent_label__isnull=False)
except Specimen.DoesNotExist:
if architect_result_row.specimen_purpose == 'panel_specimen':
error_msg += "Specimen not recognised.\n"
# only applicable to this file, can be null otherwise.
for column in self.float_columns:
value = getattr(architect_result_row, column)
try:
value = float(value)
except (ValueError, TypeError), e:
error_msg = "Could not convert column %s to float: %s" % (
column, log_exception(e))
if error_msg:
raise Exception(error_msg)
architect_result_row.state = 'validated'
architect_result_row.error_message = ''
rows_validated += 1
architect_result_row.save()
except Exception, e:
logger.exception(e)
architect_result_row.state = 'error'
architect_result_row.error_message = e.message
rows_failed += 1
architect_result_row.save()
continue
def validate(self, panel_id):
from cephia.models import Specimen, Panel, Assay
from assay.models import LuminexCDCResultRow, LuminexCDCResult, PanelMembership
rows_validated = 0
rows_failed = 0
panel = Panel.objects.get(pk=panel_id)
for luminex_result_row in LuminexCDCResultRow.objects.filter(
fileinfo=self.upload_file, state='pending'):
try:
error_msg = ''
try:
# need test_date and run for the lab
specimen = Specimen.objects.get(
specimen_label=luminex_result_row.specimen_label,
specimen_type=panel.specimen_type,
parent_label__isnull=False)
except Specimen.DoesNotExist:
if luminex_result_row.specimen_purpose == "panel_specimen":
partial_matches = Specimen.objects.partial_matches(
luminex_result_row.specimen_label,
panel.specimen_type)
if not partial_matches.count():
error_msg += "Specimen not recognised.\n"
if error_msg:
raise Exception(error_msg)
luminex_result_row.state = 'validated'
luminex_result_row.error_message = ''
rows_validated += 1
luminex_result_row.save()
except Exception, e:
logger.exception(e)
luminex_result_row.state = 'error'
luminex_result_row.error_message = log_exception(e, logger)
rows_failed += 1
luminex_result_row.save()
continue
def validate(self, panel_id):
from cephia.models import Specimen, Panel, Assay
from assay.models import GeeniusResultRow, GeeniusResult, PanelMembership
rows_validated = 0
rows_failed = 0
for geenius_result_row in GeeniusResultRow.objects.filter(
fileinfo=self.upload_file, state='pending'):
try:
error_msg = ''
panel = Panel.objects.get(pk=panel_id)
try:
Specimen.objects.get(
specimen_label=geenius_result_row.specimen_label,
specimen_type=panel.specimen_type,
parent_label__isnull=False)
except Specimen.DoesNotExist:
if geenius_result_row.specimen_purpose != 'panel_specimen':
error_msg += "Specimen not recognised.\n"
# if specimen.visit.id not in [ membership.id for membership in panel_memberhsips ]:
# error_msg += "Specimen does not belong to any panel membership.\n"
if error_msg:
raise Exception(error_msg)
geenius_result_row.state = 'validated'
geenius_result_row.error_message = ''
rows_validated += 1
geenius_result_row.save()
except Exception, e:
geenius_result_row.state = 'error'
geenius_result_row.error_message = log_exception(e, logger)
rows_failed += 1
geenius_result_row.save()
continue
def validate(self, panel_id):
from cephia.models import Specimen, Panel, Assay
from assay.models import LagSediaResultRow, LagSediaResult, PanelMembership
rows_validated = 0
rows_failed = 0
for lag_row in LagSediaResultRow.objects.filter(
fileinfo=self.upload_file, state='pending'):
try:
error_msg = ''
panel = Panel.objects.get(pk=panel_id)
try:
specimen = self.find_specimen(lag_row.specimen_label,
lag_row.specimen_purpose)
except Specimen.DoesNotExist:
if lag_row.specimen_purpose == 'panel_specimen':
# partial_matches = Specimen.objects.partial_matches(lag_row.specimen_label, panel.specimen_type)
# if not partial_matches.count():
error_msg += "Specimen not recognised.\n"
if error_msg:
raise Exception(error_msg)
lag_row.state = 'validated'
lag_row.error_message = ''
rows_validated += 1
lag_row.save()
except Exception, e:
lag_row.state = 'error'
lag_row.error_message = log_exception(e, logger)
rows_failed += 1
lag_row.save()
continue
def process(self, panel_id, assay_run):
from cephia.models import Specimen, Laboratory, Assay, Panel
from assay.models import CustomAssayResultRow, CustomAssayResult, AssayResult
rows_inserted = 0
rows_failed = 0
for custom_assay_result_row in CustomAssayResultRow.objects.filter(fileinfo=self.upload_file, state='validated'):
try:
with transaction.atomic():
assay = assay_run.assay # assay = Assay.objects.get(name=self.assay_name) which one?
panel = Panel.objects.get(pk=panel_id)
specimen = Specimen.objects.get(
specimen_label=custom_assay_result_row.specimen_label,
specimen_type=panel.specimen_type,
parent_label__isnull=False
)
test_date = None
if custom_assay_result_row.test_date:
test_date = datetime.strptime(custom_assay_result_row.test_date, '%Y-%m-%d').date()
custom_assay_result = CustomAssayResult.objects.create(
specimen=specimen,
assay=assay,
laboratory=assay_run.laboratory,
test_date=test_date,
operator=custom_assay_result_row.operator,
assay_kit_lot=custom_assay_result_row.assay_kit_lot,
plate_identifier=custom_assay_result_row.plate_identifier,
test_mode=custom_assay_result_row.test_mode,
specimen_purpose=custom_assay_result_row.specimen_purpose,
result_classification=custom_assay_result_row.result_classification,
result_quantitative=custom_assay_result_row.result_quantitative or None,
assay_run=assay_run
)
if custom_assay_result_row.result_classification is not None:
custom_assay_result.recent = custom_assay_result_row.result_classification.lower() == 'recent'
final_result = None
method = None
if 'result_quantitative' in self.header:
final_result = custom_assay_result.result_quantitative
method = None
elif custom_assay_result_row.result_classification is not None:
final_result = float(custom_assay_result.recent)
method = 'model_classification'
custom_assay_result.save()
assay_result = AssayResult.objects.create(
panel=panel,
assay=assay,
specimen=specimen,
test_date=test_date,
assay_run=assay_run,
result=final_result,
method=method
)
custom_assay_result.assay_result = assay_result
custom_assay_result.save()
custom_assay_result_row.state = 'processed'
custom_assay_result_row.date_processed = timezone.now()
custom_assay_result_row.error_message = ''
custom_assay_result_row.custom_result = custom_assay_result
custom_assay_result_row.laboratory = assay_run.laboratory.name if assay_run.laboratory else None
custom_assay_result_row.save()
rows_inserted += 1
except Exception, e:
custom_assay_result_row.state = 'error'
custom_assay_result_row.error_message = log_exception(e, logger)
custom_assay_result_row.save()
rows_failed += 1
continue
def process(self, panel_id, assay_run):
from cephia.models import Specimen, Laboratory, Assay, Panel
from assay.models import IDEV3ResultRow, IDEV3Result
rows_inserted = 0
rows_failed = 0
for ide_result_row in IDEV3ResultRow.objects.filter(
fileinfo=self.upload_file, state='validated'):
try:
with transaction.atomic():
assay = assay_run.assay
panel = Panel.objects.get(pk=panel_id)
specimen = Specimen.objects.get(
specimen_label=ide_result_row.specimen_label,
specimen_type=panel.specimen_type,
parent_label__isnull=False)
ide_result = IDEV3Result.objects.create(
specimen=specimen,
assay=assay,
laboratory=assay_run.laboratory,
test_date=datetime.strptime(ide_result_row.test_date,
'%Y-%m-%d').date(),
operator=ide_result_row.operator,
assay_kit_lot=ide_result_row.assay_kit_lot,
plate_identifier=ide_result_row.plate_identifier,
test_mode=ide_result_row.test_mode,
specimen_purpose=ide_result_row.specimen_purpose,
assay_run=assay_run,
well_tm=ide_result_row.well_tm,
well_v3=ide_result_row.well_v3,
tm_OD=ide_result_row.tm_OD,
v3_OD=ide_result_row.v3_OD,
tm_ratio_reported=ide_result_row.tm_ratio_reported,
v3_ratio_reported=ide_result_row.v3_ratio_reported,
tm_ratio=ide_result_row.tm_ratio,
v3_ratio=ide_result_row.v3_ratio,
intermediaire_reported=ide_result_row.
intermediaire_reported,
intermediaire=ide_result_row.intermediaire,
conclusion_reported=ide_result_row.conclusion_reported,
conclusion=ide_result_row.conclusion,
)
IDEV3Result.objects.get(
pk=ide_result.pk).calculate_and_save()
ide_result_row.state = 'processed'
ide_result_row.date_processed = timezone.now()
ide_result_row.error_message = ''
ide_result_row.idev3_result = ide_result
ide_result_row.save()
rows_inserted += 1
except Exception, e:
ide_result_row.state = 'error'
ide_result_row.error_message = log_exception(e, logger)
ide_result_row.save()
rows_failed += 1
continue
def process(self, panel_id, assay_run):
from cephia.models import Specimen, Laboratory, Assay, Panel
from assay.models import ISGlobalResultRow, ISGlobalResult, AssayResult
rows_inserted = 0
rows_failed = 0
for isg_result_row in ISGlobalResultRow.objects.filter(
fileinfo=self.upload_file, state='validated'):
try:
with transaction.atomic():
assay = assay_run.assay
panel = Panel.objects.get(pk=panel_id)
specimen = Specimen.objects.get(
specimen_label=isg_result_row.specimen_label,
specimen_type=panel.specimen_type,
parent_label__isnull=False)
isg_result = ISGlobalResult.objects.create(
specimen=specimen,
assay=assay,
laboratory=assay_run.laboratory,
classification_weighted_model=isg_result_row.
classification_weighted_model,
classification_unweighted_model=isg_result_row.
classification_unweighted_model,
assay_run=assay_run,
)
if isg_result_row.classification_weighted_model is not None:
isg_result.recent_weighted_model = isg_result_row.classification_weighted_model == 'Recent'
final_result = None
if isg_result_row.classification_unweighted_model is not None:
isg_result.recent_unweighted_model = isg_result_row.classification_unweighted_model == 'Recent'
final_result = float(
isg_result.recent_unweighted_model)
isg_result.save()
assay_result = AssayResult.objects.create(
panel=panel,
assay=assay,
specimen=specimen,
assay_run=assay_run,
result=final_result,
method='unweighted_model_classification')
isg_result.assay_result = assay_result
isg_result.save()
isg_result_row.state = 'processed'
isg_result_row.date_processed = timezone.now()
isg_result_row.error_message = ''
isg_result_row.isg_result = isg_result
isg_result_row.laboratory = assay_run.laboratory.name if assay_run.laboratory else None
isg_result_row.save()
rows_inserted += 1
except Exception, e:
raise
isg_result_row.state = 'error'
isg_result_row.error_message = log_exception(e, logger)
isg_result_row.save()
rows_failed += 1
continue
def process(self, panel_id, assay_run):
from cephia.models import Specimen, Laboratory, Assay, Panel
from assay.models import GeeniusResultRow, GeeniusResult, AssayResult
rows_inserted = 0
rows_failed = 0
for geenius_result_row in GeeniusResultRow.objects.filter(
fileinfo=self.upload_file, state='validated'):
try:
with transaction.atomic():
assay = Assay.objects.get(name=geenius_result_row.assay)
panel = Panel.objects.get(pk=panel_id)
specimen = Specimen.objects.get(
specimen_label=geenius_result_row.specimen_label,
specimen_type=panel.specimen_type,
parent_label__isnull=False)
warning_msg = ''
try:
is_excluded = bool(int(geenius_result_row.exclusion))
except (ValueError, TypeError):
warning_msg += 'exclusion could not be converted to an integer'
is_excluded = True
try:
final_result = float(geenius_result_row.GeeniusIndex)
except (ValueError, TypeError):
final_result = None
warning_msg += 'GeeniusIndex could not be converted to a float'
try:
result_columns = ['gp41_bi', 'gp160_bi', 'p31_bi']
result_sum = sum([
float(getattr(geenius_result_row, column))
for column in result_columns
])
final_result = result_sum / float(
geenius_result_row.ctrl_bi)
except Exception, e:
warning_msg += 'Could not recalc: ' + log_exception(e)
final_result = None
assay_result = AssayResult.objects.create(
panel=panel,
assay=assay,
specimen=specimen,
test_date=datetime.strptime(
geenius_result_row.test_date, '%Y-%m-%d').date(),
warning_msg=warning_msg,
assay_run=assay_run,
result=None if is_excluded else final_result,
method='sop_singlet')
geenius_result = GeeniusResult.objects.create(
specimen=specimen,
assay=assay,
laboratory=Laboratory.objects.get(
name=geenius_result_row.laboratory),
test_date=datetime.strptime(
geenius_result_row.test_date, '%Y-%m-%d').date(),
operator=geenius_result_row.operator,
assay_kit_lot=geenius_result_row.assay_kit_lot,
plate_identifier=geenius_result_row.plate_identifier,
test_mode=geenius_result_row.test_mode,
gp36_bi=geenius_result_row.gp36_bi,
gp140_bi=geenius_result_row.gp140_bi,
gp160_bi=geenius_result_row.gp160_bi,
p24_bi=geenius_result_row.p24_bi,
gp41_bi=geenius_result_row.gp41_bi,
ctrl_bi=geenius_result_row.ctrl_bi,
GeeniusIndex=final_result,
exclusion=geenius_result_row.exclusion,
interpretation=geenius_result_row.interpretation,
assay_result=assay_result,
assay_run=assay_run)
geenius_result_row.state = 'processed'
geenius_result_row.date_processed = timezone.now()
geenius_result_row.error_message = ''
geenius_result_row.geenius_result = geenius_result
geenius_result_row.save()
rows_inserted += 1
except Exception, e:
logger.exception(e)
geenius_result_row.state = 'error'
geenius_result_row.error_message = e.message
geenius_result_row.save()
rows_failed += 1
continue
def process(self, panel_id, assay_run):
from cephia.models import Specimen, Laboratory, Assay, Panel
from assay.models import BioPlexDukeResultRow, BioPlexDukeResult, AssayResult
rows_inserted = 0
rows_failed = 0
for duke_result_row in BioPlexDukeResultRow.objects.filter(
fileinfo=self.upload_file, state='validated'):
try:
with transaction.atomic():
assay = assay_run.assay
panel = Panel.objects.get(pk=panel_id)
specimen = Specimen.objects.get(
specimen_label=duke_result_row.specimen_label,
specimen_type=panel.specimen_type,
parent_label__isnull=False)
duke_result = BioPlexDukeResult.objects.create(
specimen=specimen,
assay=assay,
laboratory=assay_run.laboratory,
classification=duke_result_row.classification,
assay_run=assay_run,
)
if duke_result_row.classification is not None:
duke_result.recent = duke_result_row.classification == 'Recent'
final_result = None
if duke_result_row.classification is not None:
final_result = float(duke_result.recent)
duke_result.save()
assay_result = AssayResult.objects.create(
panel=panel,
assay=assay,
specimen=specimen,
assay_run=assay_run,
result=final_result,
method='model_classification')
duke_result.assay_result = assay_result
duke_result.save()
duke_result_row.state = 'processed'
duke_result_row.date_processed = timezone.now()
duke_result_row.error_message = ''
duke_result_row.duke_result = duke_result
duke_result_row.laboratory = assay_run.laboratory.name if assay_run.laboratory else None
duke_result_row.save()
rows_inserted += 1
except Exception, e:
raise
duke_result_row.state = 'error'
duke_result_row.error_message = log_exception(e, logger)
duke_result_row.save()
rows_failed += 1
continue
def process(self, panel_id, assay_run):
from cephia.models import Specimen, Laboratory, Assay, Panel
from assay.models import LagSediaResultRow, LagSediaResult, AssayResult, PanelMembership
rows_inserted = 0
rows_failed = 0
assay = Assay.objects.get(name=self.assay_name)
panel = Panel.objects.get(pk=panel_id)
panel_memberhsips = PanelMembership.objects.filter(panel=panel)
created_specimens = {}
for lag_row in LagSediaResultRow.objects.filter(
fileinfo=self.upload_file, state='validated'):
try:
warning_msg = ''
with transaction.atomic():
specimen = None
try:
specimen_hash = u'%s-%s' % (lag_row.specimen_label,
lag_row.specimen_purpose)
if specimen_hash in created_specimens:
specimen, warning_msg = created_specimens[
specimen_hash]
else:
specimen = self.find_specimen(
lag_row.specimen_label,
lag_row.specimen_purpose)
if specimen is not None:
specimen.save()
if specimen is not None and specimen.is_artificial:
warning_msg += "Artificial aliquot created"
except Specimen.DoesNotExist:
if lag_row.specimen_purpose == "panel_specimen":
raise Exception("Specimen not found")
created_specimens[specimen_hash] = (specimen, warning_msg)
lag_result = LagSediaResult.objects.create(
specimen=specimen,
assay=assay,
laboratory=Laboratory.objects.get(
name=lag_row.laboratory),
test_date=datetime.strptime(lag_row.test_date,
'%Y-%m-%d').date(),
operator=lag_row.operator,
assay_kit_lot=lag_row.assay_kit_lot,
plate_identifier=lag_row.plate_identifier,
test_mode=lag_row.test_mode,
well=lag_row.well,
specimen_purpose=lag_row.specimen_purpose,
OD=lag_row.OD or None,
calibrator_OD=lag_row.calibrator_OD or None,
ODn=lag_row.ODn or None,
assay_run=assay_run)
lag_row.state = 'processed'
lag_row.date_processed = timezone.now()
lag_row.error_message = ''
lag_row.lag_sedia_result = lag_result
lag_row.save()
rows_inserted += 1
except Exception, e:
lag_row.state = 'error'
lag_row.error_message = log_exception(e, logger)
lag_row.save()
rows_failed += 1
continue
