def parse(self): from assay.models import LagSediaResultRow rows_inserted = 0 rows_failed = 0 for row_num in range(self.num_rows): try: if row_num >= 1: self.header = [x.strip() for x in self.header] row_dict = dict(zip(self.header, self.file_rows[row_num])) lag_row = LagSediaResultRow.objects.create( specimen_label=row_dict['specimen_label'], assay=row_dict['assay'], laboratory=row_dict['laboratory'], test_date=row_dict['test_date'], operator=row_dict['operator'], assay_kit_lot=row_dict['assay_kit_lot'], plate_identifier=row_dict['plate_identifier'], test_mode=row_dict['test_mode'], well=row_dict['well'], specimen_purpose=row_dict['specimen_purpose'], OD=row_dict['OD'], calibrator_OD=row_dict['calibrator_OD'], ODn=row_dict['ODn'], state='pending', fileinfo=self.upload_file) rows_inserted += 1 except Exception, e: self.upload_file.message = "row " + str( row_num) + ": " + log_exception(e, logger) self.upload_file.save() return 0, 1
def parse(self): from assay.models import CustomAssayResultRow rows_inserted = 0 rows_failed = 0 for row_num in range(self.num_rows): try: if row_num >= 1: self.header = [ x.strip() for x in self.header ] row_dict = dict(zip(self.header, self.file_rows[row_num])) custom_assay_result_row = CustomAssayResultRow.objects.create( specimen_label = row_dict.get('specimen_label'), assay = row_dict.get('assay'), test_date = row_dict.get('test_date'), operator = row_dict.get('operator'), assay_kit_lot = row_dict.get('assay_kit_lot'), plate_identifier = row_dict.get('plate_identifier'), test_mode = row_dict.get('test_mode'), specimen_purpose=row_dict.get('specimen_purpose'), result_classification=row_dict.get('result_classification'), result_quantitative=row_dict.get('result_quantitative'), state='pending', fileinfo=self.upload_file ) rows_inserted += 1 except Exception, e: raise self.upload_file.message = "row " + str(row_num) + ": " + log_exception(e, logger) self.upload_file.save() return 0, 1
def parse(self): from assay.models import ISGlobalResultRow rows_inserted = 0 rows_failed = 0 for row_num in range(self.num_rows): try: if row_num >= 1: self.header = [x.strip() for x in self.header] row_dict = dict(zip(self.header, self.file_rows[row_num])) isg_result_row = ISGlobalResultRow.objects.create( specimen_label=row_dict['specimen_label'], specimen_purpose=row_dict['specimen_purpose'], classification_weighted_model=row_dict[ 'classification_weighted_model'], classification_unweighted_model=row_dict[ 'classification_unweighted_model'], state='pending', fileinfo=self.upload_file, assay=self.upload_file.assay, laboratory=None) rows_inserted += 1 except Exception, e: raise self.upload_file.message = "row " + str( row_num) + ": " + log_exception(e, logger) self.upload_file.save() return 0, 1
def parse(self): from assay.models import GeeniusResultRow rows_inserted = 0 rows_failed = 0 valid_columns = set(self.registered_columns) for row_num in range(self.num_rows): try: if row_num >= 1: self.header = [x.strip() for x in self.header] row_dict = dict( (k, v) for (k, v) in zip(self.header, self.file_rows[row_num]) if k in valid_columns) GeeniusResultRow.objects.create(state='pending', fileinfo=self.upload_file, **row_dict) rows_inserted += 1 except Exception, e: self.upload_file.message = "row " + str( row_num) + ": " + log_exception(e, logger) self.upload_file.save() return 0, 1
def parse(self): from assay.models import IDEV3ResultRow rows_inserted = 0 rows_failed = 0 for row_num in range(self.num_rows): try: if row_num >= 1: self.header = [x.strip() for x in self.header] row_dict = dict(zip(self.header, self.file_rows[row_num])) ide_result_row = IDEV3ResultRow.objects.create( specimen_label=row_dict['specimen_label'], assay=row_dict['assay'], laboratory=row_dict['laboratory'], test_date=row_dict['test_date'], operator=row_dict['operator'], assay_kit_lot=row_dict['assay_kit_lot'], plate_identifier=row_dict['plate_identifier'], specimen_purpose=row_dict['specimen_purpose'], exclusion=row_dict['exclusion'], interpretation=row_dict['interpretation'], test_mode=row_dict['test_mode'], well_tm=row_dict['well_tm'], well_v3=row_dict['well_v3'], tm_OD=row_dict['tm_OD'], v3_OD=row_dict['v3_OD'], tm_ratio_reported=row_dict['tm_ratio_reported'], v3_ratio_reported=row_dict['v3_ratio_reported'], tm_ratio=row_dict['tm_ratio'], v3_ratio=row_dict['v3_ratio'], intermediaire_reported=row_dict[ 'intermediaire_reported'], intermediaire=row_dict['intermediaire'], conclusion_reported=row_dict['conclusion_reported'], conclusion=row_dict['conclusion'], state='pending', fileinfo=self.upload_file) rows_inserted += 1 except Exception, e: self.upload_file.message = "row " + str( row_num) + ": " + log_exception(e, logger) self.upload_file.save() return 0, 1
def validate(self, panel_id): from cephia.models import Specimen, Panel, Assay from assay.models import ArchitectUnmodifiedResultRow, ArchitectUnmodifiedResult, PanelMembership rows_validated = 0 rows_failed = 0 for architect_result_row in ArchitectUnmodifiedResultRow.objects.filter( fileinfo=self.upload_file, state='pending'): try: error_msg = '' panel = Panel.objects.get(pk=panel_id) try: specimen = Specimen.objects.get( specimen_label=architect_result_row.specimen_label, specimen_type=panel.specimen_type, parent_label__isnull=False) except Specimen.DoesNotExist: if architect_result_row.specimen_purpose == 'panel_specimen': error_msg += "Specimen not recognised.\n" # only applicable to this file, can be null otherwise. for column in self.float_columns: value = getattr(architect_result_row, column) try: value = float(value) except (ValueError, TypeError), e: error_msg = "Could not convert column %s to float: %s" % ( column, log_exception(e)) if error_msg: raise Exception(error_msg) architect_result_row.state = 'validated' architect_result_row.error_message = '' rows_validated += 1 architect_result_row.save() except Exception, e: logger.exception(e) architect_result_row.state = 'error' architect_result_row.error_message = e.message rows_failed += 1 architect_result_row.save() continue
def validate(self, panel_id): from cephia.models import Specimen, Panel, Assay from assay.models import LuminexCDCResultRow, LuminexCDCResult, PanelMembership rows_validated = 0 rows_failed = 0 panel = Panel.objects.get(pk=panel_id) for luminex_result_row in LuminexCDCResultRow.objects.filter( fileinfo=self.upload_file, state='pending'): try: error_msg = '' try: # need test_date and run for the lab specimen = Specimen.objects.get( specimen_label=luminex_result_row.specimen_label, specimen_type=panel.specimen_type, parent_label__isnull=False) except Specimen.DoesNotExist: if luminex_result_row.specimen_purpose == "panel_specimen": partial_matches = Specimen.objects.partial_matches( luminex_result_row.specimen_label, panel.specimen_type) if not partial_matches.count(): error_msg += "Specimen not recognised.\n" if error_msg: raise Exception(error_msg) luminex_result_row.state = 'validated' luminex_result_row.error_message = '' rows_validated += 1 luminex_result_row.save() except Exception, e: logger.exception(e) luminex_result_row.state = 'error' luminex_result_row.error_message = log_exception(e, logger) rows_failed += 1 luminex_result_row.save() continue
def validate(self, panel_id): from cephia.models import Specimen, Panel, Assay from assay.models import GeeniusResultRow, GeeniusResult, PanelMembership rows_validated = 0 rows_failed = 0 for geenius_result_row in GeeniusResultRow.objects.filter( fileinfo=self.upload_file, state='pending'): try: error_msg = '' panel = Panel.objects.get(pk=panel_id) try: Specimen.objects.get( specimen_label=geenius_result_row.specimen_label, specimen_type=panel.specimen_type, parent_label__isnull=False) except Specimen.DoesNotExist: if geenius_result_row.specimen_purpose != 'panel_specimen': error_msg += "Specimen not recognised.\n" # if specimen.visit.id not in [ membership.id for membership in panel_memberhsips ]: # error_msg += "Specimen does not belong to any panel membership.\n" if error_msg: raise Exception(error_msg) geenius_result_row.state = 'validated' geenius_result_row.error_message = '' rows_validated += 1 geenius_result_row.save() except Exception, e: geenius_result_row.state = 'error' geenius_result_row.error_message = log_exception(e, logger) rows_failed += 1 geenius_result_row.save() continue
def validate(self, panel_id): from cephia.models import Specimen, Panel, Assay from assay.models import LagSediaResultRow, LagSediaResult, PanelMembership rows_validated = 0 rows_failed = 0 for lag_row in LagSediaResultRow.objects.filter( fileinfo=self.upload_file, state='pending'): try: error_msg = '' panel = Panel.objects.get(pk=panel_id) try: specimen = self.find_specimen(lag_row.specimen_label, lag_row.specimen_purpose) except Specimen.DoesNotExist: if lag_row.specimen_purpose == 'panel_specimen': # partial_matches = Specimen.objects.partial_matches(lag_row.specimen_label, panel.specimen_type) # if not partial_matches.count(): error_msg += "Specimen not recognised.\n" if error_msg: raise Exception(error_msg) lag_row.state = 'validated' lag_row.error_message = '' rows_validated += 1 lag_row.save() except Exception, e: lag_row.state = 'error' lag_row.error_message = log_exception(e, logger) rows_failed += 1 lag_row.save() continue
def process(self, panel_id, assay_run): from cephia.models import Specimen, Laboratory, Assay, Panel from assay.models import CustomAssayResultRow, CustomAssayResult, AssayResult rows_inserted = 0 rows_failed = 0 for custom_assay_result_row in CustomAssayResultRow.objects.filter(fileinfo=self.upload_file, state='validated'): try: with transaction.atomic(): assay = assay_run.assay # assay = Assay.objects.get(name=self.assay_name) which one? panel = Panel.objects.get(pk=panel_id) specimen = Specimen.objects.get( specimen_label=custom_assay_result_row.specimen_label, specimen_type=panel.specimen_type, parent_label__isnull=False ) test_date = None if custom_assay_result_row.test_date: test_date = datetime.strptime(custom_assay_result_row.test_date, '%Y-%m-%d').date() custom_assay_result = CustomAssayResult.objects.create( specimen=specimen, assay=assay, laboratory=assay_run.laboratory, test_date=test_date, operator=custom_assay_result_row.operator, assay_kit_lot=custom_assay_result_row.assay_kit_lot, plate_identifier=custom_assay_result_row.plate_identifier, test_mode=custom_assay_result_row.test_mode, specimen_purpose=custom_assay_result_row.specimen_purpose, result_classification=custom_assay_result_row.result_classification, result_quantitative=custom_assay_result_row.result_quantitative or None, assay_run=assay_run ) if custom_assay_result_row.result_classification is not None: custom_assay_result.recent = custom_assay_result_row.result_classification.lower() == 'recent' final_result = None method = None if 'result_quantitative' in self.header: final_result = custom_assay_result.result_quantitative method = None elif custom_assay_result_row.result_classification is not None: final_result = float(custom_assay_result.recent) method = 'model_classification' custom_assay_result.save() assay_result = AssayResult.objects.create( panel=panel, assay=assay, specimen=specimen, test_date=test_date, assay_run=assay_run, result=final_result, method=method ) custom_assay_result.assay_result = assay_result custom_assay_result.save() custom_assay_result_row.state = 'processed' custom_assay_result_row.date_processed = timezone.now() custom_assay_result_row.error_message = '' custom_assay_result_row.custom_result = custom_assay_result custom_assay_result_row.laboratory = assay_run.laboratory.name if assay_run.laboratory else None custom_assay_result_row.save() rows_inserted += 1 except Exception, e: custom_assay_result_row.state = 'error' custom_assay_result_row.error_message = log_exception(e, logger) custom_assay_result_row.save() rows_failed += 1 continue
def process(self, panel_id, assay_run): from cephia.models import Specimen, Laboratory, Assay, Panel from assay.models import IDEV3ResultRow, IDEV3Result rows_inserted = 0 rows_failed = 0 for ide_result_row in IDEV3ResultRow.objects.filter( fileinfo=self.upload_file, state='validated'): try: with transaction.atomic(): assay = assay_run.assay panel = Panel.objects.get(pk=panel_id) specimen = Specimen.objects.get( specimen_label=ide_result_row.specimen_label, specimen_type=panel.specimen_type, parent_label__isnull=False) ide_result = IDEV3Result.objects.create( specimen=specimen, assay=assay, laboratory=assay_run.laboratory, test_date=datetime.strptime(ide_result_row.test_date, '%Y-%m-%d').date(), operator=ide_result_row.operator, assay_kit_lot=ide_result_row.assay_kit_lot, plate_identifier=ide_result_row.plate_identifier, test_mode=ide_result_row.test_mode, specimen_purpose=ide_result_row.specimen_purpose, assay_run=assay_run, well_tm=ide_result_row.well_tm, well_v3=ide_result_row.well_v3, tm_OD=ide_result_row.tm_OD, v3_OD=ide_result_row.v3_OD, tm_ratio_reported=ide_result_row.tm_ratio_reported, v3_ratio_reported=ide_result_row.v3_ratio_reported, tm_ratio=ide_result_row.tm_ratio, v3_ratio=ide_result_row.v3_ratio, intermediaire_reported=ide_result_row. intermediaire_reported, intermediaire=ide_result_row.intermediaire, conclusion_reported=ide_result_row.conclusion_reported, conclusion=ide_result_row.conclusion, ) IDEV3Result.objects.get( pk=ide_result.pk).calculate_and_save() ide_result_row.state = 'processed' ide_result_row.date_processed = timezone.now() ide_result_row.error_message = '' ide_result_row.idev3_result = ide_result ide_result_row.save() rows_inserted += 1 except Exception, e: ide_result_row.state = 'error' ide_result_row.error_message = log_exception(e, logger) ide_result_row.save() rows_failed += 1 continue
def process(self, panel_id, assay_run): from cephia.models import Specimen, Laboratory, Assay, Panel from assay.models import ISGlobalResultRow, ISGlobalResult, AssayResult rows_inserted = 0 rows_failed = 0 for isg_result_row in ISGlobalResultRow.objects.filter( fileinfo=self.upload_file, state='validated'): try: with transaction.atomic(): assay = assay_run.assay panel = Panel.objects.get(pk=panel_id) specimen = Specimen.objects.get( specimen_label=isg_result_row.specimen_label, specimen_type=panel.specimen_type, parent_label__isnull=False) isg_result = ISGlobalResult.objects.create( specimen=specimen, assay=assay, laboratory=assay_run.laboratory, classification_weighted_model=isg_result_row. classification_weighted_model, classification_unweighted_model=isg_result_row. classification_unweighted_model, assay_run=assay_run, ) if isg_result_row.classification_weighted_model is not None: isg_result.recent_weighted_model = isg_result_row.classification_weighted_model == 'Recent' final_result = None if isg_result_row.classification_unweighted_model is not None: isg_result.recent_unweighted_model = isg_result_row.classification_unweighted_model == 'Recent' final_result = float( isg_result.recent_unweighted_model) isg_result.save() assay_result = AssayResult.objects.create( panel=panel, assay=assay, specimen=specimen, assay_run=assay_run, result=final_result, method='unweighted_model_classification') isg_result.assay_result = assay_result isg_result.save() isg_result_row.state = 'processed' isg_result_row.date_processed = timezone.now() isg_result_row.error_message = '' isg_result_row.isg_result = isg_result isg_result_row.laboratory = assay_run.laboratory.name if assay_run.laboratory else None isg_result_row.save() rows_inserted += 1 except Exception, e: raise isg_result_row.state = 'error' isg_result_row.error_message = log_exception(e, logger) isg_result_row.save() rows_failed += 1 continue
def process(self, panel_id, assay_run): from cephia.models import Specimen, Laboratory, Assay, Panel from assay.models import GeeniusResultRow, GeeniusResult, AssayResult rows_inserted = 0 rows_failed = 0 for geenius_result_row in GeeniusResultRow.objects.filter( fileinfo=self.upload_file, state='validated'): try: with transaction.atomic(): assay = Assay.objects.get(name=geenius_result_row.assay) panel = Panel.objects.get(pk=panel_id) specimen = Specimen.objects.get( specimen_label=geenius_result_row.specimen_label, specimen_type=panel.specimen_type, parent_label__isnull=False) warning_msg = '' try: is_excluded = bool(int(geenius_result_row.exclusion)) except (ValueError, TypeError): warning_msg += 'exclusion could not be converted to an integer' is_excluded = True try: final_result = float(geenius_result_row.GeeniusIndex) except (ValueError, TypeError): final_result = None warning_msg += 'GeeniusIndex could not be converted to a float' try: result_columns = ['gp41_bi', 'gp160_bi', 'p31_bi'] result_sum = sum([ float(getattr(geenius_result_row, column)) for column in result_columns ]) final_result = result_sum / float( geenius_result_row.ctrl_bi) except Exception, e: warning_msg += 'Could not recalc: ' + log_exception(e) final_result = None assay_result = AssayResult.objects.create( panel=panel, assay=assay, specimen=specimen, test_date=datetime.strptime( geenius_result_row.test_date, '%Y-%m-%d').date(), warning_msg=warning_msg, assay_run=assay_run, result=None if is_excluded else final_result, method='sop_singlet') geenius_result = GeeniusResult.objects.create( specimen=specimen, assay=assay, laboratory=Laboratory.objects.get( name=geenius_result_row.laboratory), test_date=datetime.strptime( geenius_result_row.test_date, '%Y-%m-%d').date(), operator=geenius_result_row.operator, assay_kit_lot=geenius_result_row.assay_kit_lot, plate_identifier=geenius_result_row.plate_identifier, test_mode=geenius_result_row.test_mode, gp36_bi=geenius_result_row.gp36_bi, gp140_bi=geenius_result_row.gp140_bi, gp160_bi=geenius_result_row.gp160_bi, p24_bi=geenius_result_row.p24_bi, gp41_bi=geenius_result_row.gp41_bi, ctrl_bi=geenius_result_row.ctrl_bi, GeeniusIndex=final_result, exclusion=geenius_result_row.exclusion, interpretation=geenius_result_row.interpretation, assay_result=assay_result, assay_run=assay_run) geenius_result_row.state = 'processed' geenius_result_row.date_processed = timezone.now() geenius_result_row.error_message = '' geenius_result_row.geenius_result = geenius_result geenius_result_row.save() rows_inserted += 1 except Exception, e: logger.exception(e) geenius_result_row.state = 'error' geenius_result_row.error_message = e.message geenius_result_row.save() rows_failed += 1 continue
def process(self, panel_id, assay_run): from cephia.models import Specimen, Laboratory, Assay, Panel from assay.models import BioPlexDukeResultRow, BioPlexDukeResult, AssayResult rows_inserted = 0 rows_failed = 0 for duke_result_row in BioPlexDukeResultRow.objects.filter( fileinfo=self.upload_file, state='validated'): try: with transaction.atomic(): assay = assay_run.assay panel = Panel.objects.get(pk=panel_id) specimen = Specimen.objects.get( specimen_label=duke_result_row.specimen_label, specimen_type=panel.specimen_type, parent_label__isnull=False) duke_result = BioPlexDukeResult.objects.create( specimen=specimen, assay=assay, laboratory=assay_run.laboratory, classification=duke_result_row.classification, assay_run=assay_run, ) if duke_result_row.classification is not None: duke_result.recent = duke_result_row.classification == 'Recent' final_result = None if duke_result_row.classification is not None: final_result = float(duke_result.recent) duke_result.save() assay_result = AssayResult.objects.create( panel=panel, assay=assay, specimen=specimen, assay_run=assay_run, result=final_result, method='model_classification') duke_result.assay_result = assay_result duke_result.save() duke_result_row.state = 'processed' duke_result_row.date_processed = timezone.now() duke_result_row.error_message = '' duke_result_row.duke_result = duke_result duke_result_row.laboratory = assay_run.laboratory.name if assay_run.laboratory else None duke_result_row.save() rows_inserted += 1 except Exception, e: raise duke_result_row.state = 'error' duke_result_row.error_message = log_exception(e, logger) duke_result_row.save() rows_failed += 1 continue
def process(self, panel_id, assay_run): from cephia.models import Specimen, Laboratory, Assay, Panel from assay.models import LagSediaResultRow, LagSediaResult, AssayResult, PanelMembership rows_inserted = 0 rows_failed = 0 assay = Assay.objects.get(name=self.assay_name) panel = Panel.objects.get(pk=panel_id) panel_memberhsips = PanelMembership.objects.filter(panel=panel) created_specimens = {} for lag_row in LagSediaResultRow.objects.filter( fileinfo=self.upload_file, state='validated'): try: warning_msg = '' with transaction.atomic(): specimen = None try: specimen_hash = u'%s-%s' % (lag_row.specimen_label, lag_row.specimen_purpose) if specimen_hash in created_specimens: specimen, warning_msg = created_specimens[ specimen_hash] else: specimen = self.find_specimen( lag_row.specimen_label, lag_row.specimen_purpose) if specimen is not None: specimen.save() if specimen is not None and specimen.is_artificial: warning_msg += "Artificial aliquot created" except Specimen.DoesNotExist: if lag_row.specimen_purpose == "panel_specimen": raise Exception("Specimen not found") created_specimens[specimen_hash] = (specimen, warning_msg) lag_result = LagSediaResult.objects.create( specimen=specimen, assay=assay, laboratory=Laboratory.objects.get( name=lag_row.laboratory), test_date=datetime.strptime(lag_row.test_date, '%Y-%m-%d').date(), operator=lag_row.operator, assay_kit_lot=lag_row.assay_kit_lot, plate_identifier=lag_row.plate_identifier, test_mode=lag_row.test_mode, well=lag_row.well, specimen_purpose=lag_row.specimen_purpose, OD=lag_row.OD or None, calibrator_OD=lag_row.calibrator_OD or None, ODn=lag_row.ODn or None, assay_run=assay_run) lag_row.state = 'processed' lag_row.date_processed = timezone.now() lag_row.error_message = '' lag_row.lag_sedia_result = lag_result lag_row.save() rows_inserted += 1 except Exception, e: lag_row.state = 'error' lag_row.error_message = log_exception(e, logger) lag_row.save() rows_failed += 1 continue