Example #1
0
def _create_sampleSetItem(csvSampleDict, request, user, sampleSet_ids):
    sampleDisplayedName = csvSampleDict.get(COLUMN_SAMPLE_NAME, '').strip()
    sampleExtId = csvSampleDict.get(COLUMN_SAMPLE_EXT_ID, '').strip()
    sampleGender = csvSampleDict.get(COLUMN_GENDER, '').strip()
    sampleControlType = csvSampleDict.get(COLUMN_CONTROLTYPE, '').strip()
    sampleGroupType = csvSampleDict.get(COLUMN_GROUP_TYPE, None)
    sampleGroup = csvSampleDict.get(COLUMN_GROUP, '0').strip()
    sampleDescription = csvSampleDict.get(COLUMN_SAMPLE_DESCRIPTION, '').strip()
    barcodeKit = csvSampleDict.get(COLUMN_BARCODE_KIT, '').strip()
    barcodeAssignment = csvSampleDict.get(COLUMN_BARCODE, '').strip()
    nucleotideType = csvSampleDict.get(COLUMN_NUCLEOTIDE_TYPE, "").strip() or csvSampleDict.get(ALTERNATE_COLUMN_NUCLEOTIDE_TYPE, "").strip()

    cancerType = csvSampleDict.get(COLUMN_CANCER_TYPE, "").strip()
    cellularityPct = csvSampleDict.get(COLUMN_CELLULARITY_PCT, None).strip()
    pcrPlateRow = csvSampleDict.get(COLUMN_PCR_PLATE_POSITION, "").strip()

    biopsyDays = csvSampleDict.get(COLUMN_BIOPSY_DAYS, "0").strip()
    cellNum = csvSampleDict.get(COLUMN_CELL_NUM, "").strip()
    coupleId = csvSampleDict.get(COLUMN_COUPLE_ID, None).strip()
    embryoId = csvSampleDict.get(COLUMN_EMBRYO_ID, "").strip()

    if not sampleGroup:
        sampleGroup = '0'

    isValid, errorMessage, nucleotideType_internal_value = sample_validator.validate_nucleotideType(nucleotideType)

    # validation has been done already, this is just to get the official value
    isValid, errorMessage, gender_CV_value = sample_validator.validate_sampleGender(sampleGender)
    isValid, errorMessage, role_CV_value = sample_validator.validate_sampleGroupType(sampleGroupType)
    isValid, errorMessage, controlType_CV_value = sample_validator.validate_controlType(sampleControlType)
    isValid, errorMessage, cancerType_CV_value = sample_validator.validate_cancerType(cancerType)
    isValid, errorMessage, pcrPlateRow_internal_value = sample_validator.validate_pcrPlateRow(pcrPlateRow)

    sampleName = sampleDisplayedName.replace(' ', '_')
    sample_kwargs = {
        'displayedName': sampleDisplayedName,
        'status': 'created',
        'description': sampleDescription,
        'date': timezone.now()  ##datetime.datetime.now()
    }

    sample, isCreated = Sample.objects.get_or_create(name=sampleName, externalId=sampleExtId, defaults=sample_kwargs)

    if isCreated:
        logger.debug("import_sample_processor._create_sampleSetItem() new sample created for sample=%s; id=%d" % (sampleDisplayedName, sample.id))
    else:
        if (sample.description != sampleDescription):
            sample.description = sampleDescription
            sample.save()

            logger.debug("import_sample_processor._create_sampleSetItem() just updated sample description for sample=%s; id=%d" % (sampleDisplayedName, sample.id))

    # logger.debug("import_sample_processor._create_sampleSetItem() after get_or_create isCreated=%s; sample=%s; sample.id=%d" %(str(isCreated), sampleDisplayedName, sample.id))

    for sampleSetId in sampleSet_ids:
        logger.debug("import_sample_processor._create_sampleSetItem() going to create sampleSetItem for sample=%s; sampleSetId=%s in sampleSet_ids=%s" % (sampleDisplayedName, str(sampleSetId), sampleSet_ids))

        currentDateTime = timezone.now()  ##datetime.datetime.now()

        dnabarcode = None
        if barcodeKit and barcodeAssignment:
            dnabarcode = models.dnaBarcode.objects.get(name__iexact=barcodeKit, id_str__iexact=barcodeAssignment)

        pcrPlateColumn = "1" if pcrPlateRow_internal_value else ""

        sampleSetItem_kwargs = {
            'gender': gender_CV_value,
            'relationshipRole': role_CV_value,
            'relationshipGroup': sampleGroup,
            'cancerType': cancerType_CV_value,
            'cellularityPct': cellularityPct if cellularityPct else None,
            'biopsyDays': int(biopsyDays) if biopsyDays else 0,
            "cellNum": cellNum,
            "coupleId": coupleId,
            "embryoId": embryoId,
            'creator': user,
            'creationDate': currentDateTime,
            'lastModifiedUser': user,
            'lastModifiedDate': currentDateTime,
            'description': sampleDescription,
            'controlType': controlType_CV_value
        }

        sampleSetItem, isCreated = SampleSetItem.objects.get_or_create(sample=sample,
                                                                       sampleSet_id=sampleSetId,
                                                                       dnabarcode=dnabarcode,
                                                                       description=sampleDescription,
                                                                       nucleotideType=nucleotideType_internal_value,
                                                                       pcrPlateRow=pcrPlateRow_internal_value,
                                                                       pcrPlateColumn=pcrPlateColumn,
                                                                       defaults=sampleSetItem_kwargs)

        logger.debug("import_sample_processor._create_sampleSetItem() after get_or_create isCreated=%s; sampleSetItem=%s; samplesetItem.id=%d" % (str(isCreated), sampleDisplayedName, sampleSetItem.id))

    ssi_sid = transaction.savepoint()

    return sample, sampleSetItem, ssi_sid
def validate_csv_sample(csvSampleDict, request):
    """ 
    validate csv contents and convert user input to raw data to prepare for sample persistence
    returns: a collection of error messages if errors found and whether to skip the row
    """
    failed = []
    isToSkipRow = False
    isToAbort = False

    logger.debug("ENTER import_sample_processor.validate_csv_sample() csvSampleDict=%s; " %(csvSampleDict))
    
    try:
        sampleDisplayedName = csvSampleDict.get(COLUMN_SAMPLE_NAME, '').strip()
        sampleExtId = csvSampleDict.get(COLUMN_SAMPLE_EXT_ID, '').strip()
        sampleGender = csvSampleDict.get(COLUMN_GENDER, '').strip()
        sampleGroupType = csvSampleDict.get(COLUMN_GROUP_TYPE, '').strip()
        sampleGroup = csvSampleDict.get(COLUMN_GROUP, '').strip()
        sampleDescription = csvSampleDict.get(COLUMN_SAMPLE_DESCRIPTION, '').strip()
        barcodeKit = csvSampleDict.get(COLUMNS_BARCODE_KIT, '').strip()
        barcodeAssignment = csvSampleDict.get(COLUMN_BARCODE, '').strip()
        
        nucleotideType = csvSampleDict.get(COLUMN_NUCLEOTIDE_TYPE, "").strip()
     
        cancerType = csvSampleDict.get(COLUMN_CANCER_TYPE, "").strip()
        cellularityPct = csvSampleDict.get(COLUMN_CELLULARITY_PCT, None).strip()
        pcrPlateRow = csvSampleDict.get(COLUMN_PCR_PLATE_POSITION, "").strip()
        
        #skip blank line
        hasAtLeastOneValue = bool([v for v in csvSampleDict.values() if v != ''])
        if not hasAtLeastOneValue:
            isToSkipRow = True
            return failed, isToSkipRow, isToAbort
        
        isValid, errorMessage = sample_validator.validate_sampleDisplayedName(sampleDisplayedName)
        if not isValid:
            failed.append((COLUMN_SAMPLE_NAME, errorMessage))    
            
        isValid, errorMessage = sample_validator.validate_sampleExternalId(sampleExtId)
        if not isValid:
            failed.append((COLUMN_SAMPLE_EXT_ID, errorMessage))
            
        isValid, errorMessage = sample_validator.validate_sampleDescription(sampleDescription)
        if not isValid:
            failed.append((COLUMN_SAMPLE_DESCRIPTION, errorMessage))
            
        isValid, errorMessage, gender_CV_value = sample_validator.validate_sampleGender(sampleGender)
        if not isValid:
            failed.append((COLUMN_GENDER, errorMessage))
           
        isValid, errorMessage, role_CV_value = sample_validator.validate_sampleGroupType(sampleGroupType)
        if not isValid:
            failed.append((COLUMN_GROUP_TYPE, errorMessage))
            
        if sampleGroup: 
            isValid, errorMessage = sample_validator.validate_sampleGroup(sampleGroup)
            if not isValid:
                failed.append((COLUMN_GROUP, errorMessage))
    
        if cancerType:
            isValid, errorMessage, cancerType_CV_value = sample_validator.validate_cancerType(cancerType) 
            if not isValid:
                failed.append((COLUMN_CANCER_TYPE, errorMessage))
        
        if cellularityPct:
            isValid, errorMessage, value = sample_validator.validate_cellularityPct(cellularityPct)
            if not isValid:
                failed.append((COLUMN_CELLULARITY_PCT, errorMessage))     

        if nucleotideType:
            isValid, errorMessage, nucleotideType_internal_value = sample_validator.validate_nucleotideType(nucleotideType)
            if not isValid:
                failed.append((COLUMN_NUCLEOTIDE_TYPE, errorMessage))     
        
        if pcrPlateRow:
            isValid, errorMessage, pcrPlateRow_internal_value = sample_validator.validate_pcrPlateRow(pcrPlateRow)
            if not isValid:
                failed.append((COLUMN_PCR_PLATE_POSITION, errorMessage))     

        ##NEW VALIDATION FOR BARCODEKIT AND BARCODE_ID_STR
        isValid, errorMessage, item = sample_validator.validate_barcodekit_and_id_str(barcodeKit, barcodeAssignment)
        if not isValid:
            if item == 'barcodeKit':
                failed.append((COLUMNS_BARCODE_KIT, errorMessage))
            else:
                failed.append((COLUMN_BARCODE, errorMessage))
        # if not isValid:
        #     failed.append((COLUMN_BARCODE, errorMessage))
    
        #validate user-defined custom attributes
        failed_userDefined = _validate_csv_user_defined_attributes(csvSampleDict, request)
        failed.extend(failed_userDefined)
                   
        logger.debug("import_sample_processor.validate_csv_sample() failed=%s" %(failed))
            
        return failed, isToSkipRow, isToAbort
    
    except:
        logger.exception(format_exc())
        
        failed.append(("File Contents", " the CSV file does not seem to have all the columns. Click the Sample File Format button for an example. "))
        isToAbort = True               
        logger.debug("import_sample_processor.validate_csv_sample() failed=%s" %(failed))        
        return failed, isToSkipRow, isToAbort
Example #3
0
def validate_csv_sample(csvSampleDict, request):
    """
    validate csv contents and convert user input to raw data to prepare for sample persistence
    returns: a collection of error messages if errors found and whether to skip the row
    """
    failed = []
    isToSkipRow = False
    isToAbort = False

    logger.debug("ENTER import_sample_processor.validate_csv_sample() csvSampleDict=%s; " % (csvSampleDict))

    try:
        sampleDisplayedName = csvSampleDict.get(COLUMN_SAMPLE_NAME, '').strip()
        sampleExtId = csvSampleDict.get(COLUMN_SAMPLE_EXT_ID, '').strip()
        sampleControlType = csvSampleDict.get(COLUMN_CONTROLTYPE, '').strip()
        sampleGender = csvSampleDict.get(COLUMN_GENDER, '').strip()
        sampleGroupType = csvSampleDict.get(COLUMN_GROUP_TYPE, '').strip()
        sampleGroup = csvSampleDict.get(COLUMN_GROUP, '').strip()
        sampleDescription = csvSampleDict.get(COLUMN_SAMPLE_DESCRIPTION, '').strip()
        barcodeKit = csvSampleDict.get(COLUMN_BARCODE_KIT, '')
        barcodeAssignment = csvSampleDict.get(COLUMN_BARCODE, '')

        nucleotideType = csvSampleDict.get(COLUMN_NUCLEOTIDE_TYPE, "").strip()

        cancerType = csvSampleDict.get(COLUMN_CANCER_TYPE, "").strip()
        cellularityPct = csvSampleDict.get(COLUMN_CELLULARITY_PCT, None).strip()
        pcrPlateRow = csvSampleDict.get(COLUMN_PCR_PLATE_POSITION, "").strip()

        biopsyDays = csvSampleDict.get(COLUMN_BIOPSY_DAYS, "0").strip()
        cellNum = csvSampleDict.get(COLUMN_CELL_NUM, "").strip()
        coupleId = csvSampleDict.get(COLUMN_COUPLE_ID, "").strip()
        embryoId = csvSampleDict.get(COLUMN_EMBRYO_ID, "").strip()

        # Trim off barcode and barcode kit leading and trailing spaces and update the log file if exists
        if ((len(barcodeKit) - len(barcodeKit.lstrip())) or
                (len(barcodeKit) - len(barcodeKit.rstrip()))):
            logger.warning("The BarcodeKitName(%s) contains Leading/Trailing spaces and got trimmed." % barcodeKit)

        if ((len(barcodeAssignment) - len(barcodeAssignment.lstrip())) or
                (len(barcodeAssignment) - len(barcodeAssignment.rstrip()))):
            logger.warning("The BarcodeName (%s) of BarcodeKitName(%s) contains Leading/Trailing spaces and got trimmed." % (barcodeAssignment, barcodeKit))

        barcodeKit = barcodeKit.strip()
        barcodeAssignment = barcodeAssignment.strip()

        # skip blank line
        hasAtLeastOneValue = bool([v for v in csvSampleDict.values() if v != ''])
        if not hasAtLeastOneValue:
            isToSkipRow = True
            return failed, isToSkipRow, isToAbort

        isValid, errorMessage = sample_validator.validate_sampleDisplayedName(sampleDisplayedName)
        if not isValid:
            failed.append((COLUMN_SAMPLE_NAME, errorMessage))

        isValid, errorMessage = sample_validator.validate_sampleExternalId(sampleExtId)
        if not isValid:
            failed.append((COLUMN_SAMPLE_EXT_ID, errorMessage))

        isValid, errorMessage = sample_validator.validate_sampleDescription(sampleDescription)
        if not isValid:
            failed.append((COLUMN_SAMPLE_DESCRIPTION, errorMessage))

        isValid, errorMessage, gender_CV_value = sample_validator.validate_sampleGender(sampleGender)
        if not isValid:
            failed.append((COLUMN_GENDER, errorMessage))

        isValid, errorMessage, role_CV_value = sample_validator.validate_sampleGroupType(sampleGroupType)
        if not isValid:
            failed.append((COLUMN_GROUP_TYPE, errorMessage))

        if sampleGroup:
            isValid, errorMessage = sample_validator.validate_sampleGroup(sampleGroup)
            if not isValid:
                failed.append((COLUMN_GROUP, errorMessage))

        if cancerType:
            isValid, errorMessage, cancerType_CV_value = sample_validator.validate_cancerType(cancerType)
            if not isValid:
                failed.append((COLUMN_CANCER_TYPE, errorMessage))

        if cellularityPct:
            isValid, errorMessage, value = sample_validator.validate_cellularityPct(cellularityPct)
            if not isValid:
                failed.append((COLUMN_CELLULARITY_PCT, errorMessage))

        if nucleotideType:
            isValid, errorMessage, nucleotideType_internal_value = sample_validator.validate_nucleotideType(nucleotideType)
            if not isValid:
                failed.append((COLUMN_NUCLEOTIDE_TYPE, errorMessage))

        if pcrPlateRow:
            isValid, errorMessage, pcrPlateRow_internal_value = sample_validator.validate_pcrPlateRow(pcrPlateRow)
            if not isValid:
                failed.append((COLUMN_PCR_PLATE_POSITION, errorMessage))

        if biopsyDays:
            isValid, errorMessage = sample_validator.validate_sampleBiopsyDays(biopsyDays)
            if not isValid:
                failed.append((COLUMN_BIOPSY_DAYS, errorMessage))

        if cellNum:
            isValid, errorMessage = sample_validator.validate_sampleCellNum(cellNum)
            if not isValid:
                failed.append((COLUMN_CELL_NUM, errorMessage))

        if coupleId:
            isValid, errorMessage = sample_validator.validate_sampleCoupleId(coupleId)
            if not isValid:
                failed.append((COLUMN_COUPLE_ID, errorMessage))

        if embryoId:
            isValid, errorMessage = sample_validator.validate_sampleEmbryoId(embryoId)
            if not isValid:
                failed.append((COLUMN_EMBRYO_ID, errorMessage))

        if sampleControlType:
            isValid, errorMessage, controlType_CV_value = sample_validator.validate_controlType(sampleControlType)
            if not isValid:
                failed.append((COLUMN_CONTROLTYPE, errorMessage))

        # NEW VALIDATION FOR BARCODEKIT AND BARCODE_ID_STR
        isValid, errorMessage, item = sample_validator.validate_barcodekit_and_id_str(barcodeKit, barcodeAssignment)
        if not isValid:
            if item == 'barcodeKit':
                failed.append((COLUMN_BARCODE_KIT, errorMessage))
            else:
                failed.append((COLUMN_BARCODE, errorMessage))
        # if not isValid:
        #     failed.append((COLUMN_BARCODE, errorMessage))

        # validate user-defined custom attributes
        failed_userDefined = _validate_csv_user_defined_attributes(csvSampleDict, request)
        failed.extend(failed_userDefined)

        logger.debug("import_sample_processor.validate_csv_sample() failed=%s" % (failed))

        return failed, isToSkipRow, isToAbort

    except:
        logger.exception(format_exc())

        failed.append(("File Contents", " the CSV file does not seem to have all the columns. Click the Sample File Format button for an example. "))
        isToAbort = True
        logger.debug("import_sample_processor.validate_csv_sample() failed=%s" % (failed))
        return failed, isToSkipRow, isToAbort
Example #4
0
def get_sampleSetItem_kwargs(csvSampleDict, user):
    sampleDisplayedName = csvSampleDict.get(COLUMN_SAMPLE_NAME, "").strip()
    sampleExtId = csvSampleDict.get(COLUMN_SAMPLE_EXT_ID, "").strip()
    sampleGender = csvSampleDict.get(COLUMN_GENDER, "").strip()
    sampleControlType = csvSampleDict.get(COLUMN_CONTROLTYPE, "").strip()
    sampleGroupType = csvSampleDict.get(COLUMN_GROUP_TYPE, None)
    sampleGroup = csvSampleDict.get(COLUMN_GROUP, "0").strip()
    if not sampleGroup:
        sampleGroup = "0"
    sampleDescription = csvSampleDict.get(COLUMN_SAMPLE_DESCRIPTION,
                                          "").strip()
    barcodeKit = csvSampleDict.get(COLUMN_BARCODE_KIT, "").strip()
    barcodeAssignment = csvSampleDict.get(COLUMN_BARCODE, "").strip()
    pcrPlateRow = csvSampleDict.get(COLUMN_PCR_PLATE_POSITION, "").strip()
    nucleotideType = (csvSampleDict.get(COLUMN_NUCLEOTIDE_TYPE, "").strip()
                      or csvSampleDict.get(ALTERNATE_COLUMN_NUCLEOTIDE_TYPE,
                                           "").strip())
    sampleSource = csvSampleDict.get(COLUMN_SAMPLE_SOURCE, "").strip()
    panelPoolType = csvSampleDict.get(COLUMN_PANEL_POOL_TYPE, "").strip()
    sampleCollectionDate = csvSampleDict.get(COLUMN_SAMPLE_COLLECTION_DATE,
                                             "").strip()
    sampleReceiptDate = csvSampleDict.get(COLUMN_SAMPLE_RECEIPT_DATE,
                                          "").strip()
    if sampleCollectionDate:
        sampleCollectionDate = datetime.strptime(str(sampleCollectionDate),
                                                 "%Y-%m-%d").date()
    if sampleReceiptDate:
        sampleReceiptDate = datetime.strptime(str(sampleReceiptDate),
                                              "%Y-%m-%d").date()

    cancerType = csvSampleDict.get(COLUMN_CANCER_TYPE, "").strip()
    cellularityPct = csvSampleDict.get(COLUMN_CELLULARITY_PCT, None)
    if cellularityPct:
        cellularityPct = cellularityPct.strip()

    if not cellularityPct:
        cellularityPct = None

    biopsyDays = csvSampleDict.get(COLUMN_BIOPSY_DAYS, "0").strip()
    cellNum = csvSampleDict.get(COLUMN_CELL_NUM, "").strip()
    coupleId = csvSampleDict.get(COLUMN_COUPLE_ID, None)
    if coupleId:
        coupleId = coupleId.strip()
    embryoId = csvSampleDict.get(COLUMN_EMBRYO_ID, "").strip()
    population = csvSampleDict.get(COLUMN_SAMPLE_POPULATION, "").strip()
    mouseStrains = csvSampleDict.get(COLUMN_SAMPLE_MOUSE_STRAINS, "").strip()
    currentDateTime = timezone.now()  ##datetime.datetime.now()

    sampleName = sampleDisplayedName.replace(" ", "_")

    isValid, errorMessage, nucleotideType_internal_value = sample_validator.validate_nucleotideType(
        nucleotideType, field_label=COLUMN_NUCLEOTIDE_TYPE)

    # validation has been done already, this is just to get the official value
    isValid, errorMessage, gender_CV_value = sample_validator.validate_sampleGender(
        sampleGender, field_label=COLUMN_GENDER)
    isValid, errorMessage, role_CV_value = sample_validator.validate_sampleGroupType(
        sampleGroupType, field_label=COLUMN_GROUP_TYPE)
    isValid, errorMessage, controlType_CV_value = sample_validator.validate_controlType(
        sampleControlType, field_label=COLUMN_CONTROLTYPE)
    isValid, errorMessage, cancerType_CV_value = sample_validator.validate_cancerType(
        cancerType, field_label=COLUMN_CANCER_TYPE)
    isValid, errorMessage, pcrPlateRow_internal_value = sample_validator.validate_pcrPlateRow(
        pcrPlateRow, field_label=COLUMN_PCR_PLATE_POSITION)

    sampleSetItem_kwargs = {
        "sampleName": sampleName,
        "sampleDisplayedName": sampleDisplayedName,
        "sampleExtId": sampleExtId,
        "barcodeKit": barcodeKit,
        "barcodeAssignment": barcodeAssignment,
        "gender": gender_CV_value,
        "relationshipRole": role_CV_value,
        "relationshipGroup": sampleGroup,
        "cancerType": cancerType_CV_value,
        "pcrPlateRow": pcrPlateRow_internal_value,
        "nucleotideType": nucleotideType_internal_value,
        "sampleSource": sampleSource,
        "panelPoolType": panelPoolType,
        "cellularityPct": cellularityPct if cellularityPct else None,
        "biopsyDays": int(biopsyDays) if biopsyDays else 0,
        "cellNum": cellNum,
        "coupleId": coupleId,
        "embryoId": embryoId,
        "creator": user,
        "creationDate": currentDateTime,
        "lastModifiedUser": user,
        "lastModifiedDate": currentDateTime,
        "description": sampleDescription,
        "controlType": controlType_CV_value,
        "displayedName": sampleDisplayedName,
        "sampleStatus": "created",
        "sampleDescription": sampleDescription,
        "sampleCollectionDate": sampleCollectionDate or None,
        "sampleReceiptDate": sampleReceiptDate or None,
        "population": population,
        "mouseStrains": mouseStrains,
        "date": timezone.now(),
    }

    return sampleSetItem_kwargs
Example #5
0
def validate_csv_sample(csvSampleDict, request):
    """
    validate csv contents and convert user input to raw data to prepare for sample persistence
    returns: a collection of error messages if errors found and whether to skip the row
    """
    failed = []
    isToSkipRow = False
    isToAbort = False

    logger.debug(
        "ENTER import_sample_processor.validate_csv_sample() csvSampleDict=%s; "
        % (csvSampleDict))
    try:

        sampleDisplayedName = csvSampleDict.get(COLUMN_SAMPLE_NAME, "").strip()
        sampleExtId = csvSampleDict.get(COLUMN_SAMPLE_EXT_ID, "").strip()
        sampleControlType = csvSampleDict.get(COLUMN_CONTROLTYPE, "").strip()
        sampleGender = csvSampleDict.get(COLUMN_GENDER, "").strip()
        sampleGroupType = csvSampleDict.get(COLUMN_GROUP_TYPE, "").strip()
        sampleGroup = csvSampleDict.get(COLUMN_GROUP, "").strip()
        if not sampleGroup:
            sampleGroup = None
        sampleDescription = csvSampleDict.get(COLUMN_SAMPLE_DESCRIPTION,
                                              "").strip()
        barcodeKit = csvSampleDict.get(COLUMN_BARCODE_KIT, "")
        barcodeAssignment = csvSampleDict.get(COLUMN_BARCODE, "")

        nucleotideType = csvSampleDict.get(COLUMN_NUCLEOTIDE_TYPE, "").strip()

        cancerType = csvSampleDict.get(COLUMN_CANCER_TYPE, "").strip()
        cellularityPct = csvSampleDict.get(COLUMN_CELLULARITY_PCT,
                                           None).strip()
        if not cellularityPct:
            cellularityPct = None
        pcrPlateRow = csvSampleDict.get(COLUMN_PCR_PLATE_POSITION, "").strip()

        biopsyDays = csvSampleDict.get(COLUMN_BIOPSY_DAYS, "0").strip()
        cellNum = csvSampleDict.get(COLUMN_CELL_NUM, "").strip()
        coupleId = csvSampleDict.get(COLUMN_COUPLE_ID, "").strip()
        embryoId = csvSampleDict.get(COLUMN_EMBRYO_ID, "").strip()
        sampleSource = csvSampleDict.get(COLUMN_SAMPLE_SOURCE, "").strip()
        panelPoolType = csvSampleDict.get(COLUMN_PANEL_POOL_TYPE, "").strip()
        sampleCollectionDate = csvSampleDict.get(COLUMN_SAMPLE_COLLECTION_DATE,
                                                 "").strip()
        sampleReceiptDate = csvSampleDict.get(COLUMN_SAMPLE_RECEIPT_DATE,
                                              "").strip()
        population = csvSampleDict.get(COLUMN_SAMPLE_POPULATION, "").strip()
        mouseStrains = csvSampleDict.get(COLUMN_SAMPLE_MOUSE_STRAINS,
                                         "").strip()

        # Trim off barcode and barcode kit leading and trailing spaces and update the log file if exists
        if (len(barcodeKit) - len(barcodeKit.lstrip())) or (
                len(barcodeKit) - len(barcodeKit.rstrip())):
            logger.warning(
                "The BarcodeKitName(%s) contains Leading/Trailing spaces and got trimmed."
                % barcodeKit)

        if (len(barcodeAssignment) - len(barcodeAssignment.lstrip())) or (
                len(barcodeAssignment) - len(barcodeAssignment.rstrip())):
            logger.warning(
                "The BarcodeName (%s) of BarcodeKitName(%s) contains Leading/Trailing spaces and got trimmed."
                % (barcodeAssignment, barcodeKit))

        barcodeKit = barcodeKit.strip()
        barcodeAssignment = barcodeAssignment.strip()

        # skip blank line
        hasAtLeastOneValue = bool(
            [v for v in list(csvSampleDict.values()) if v != ""])
        if not hasAtLeastOneValue:
            isToSkipRow = True
            return failed, isToSkipRow, isToAbort

        isValid, errorMessage = sample_validator.validate_sampleDisplayedName(
            sampleDisplayedName, field_label=COLUMN_SAMPLE_NAME)
        if not isValid:
            failed.append((COLUMN_SAMPLE_NAME, errorMessage))

        isValid, errorMessage = sample_validator.validate_sampleExternalId(
            sampleExtId, field_label=COLUMN_SAMPLE_EXT_ID)
        if not isValid:
            failed.append((COLUMN_SAMPLE_EXT_ID, errorMessage))

        isValid, errorMessage = sample_validator.validate_sampleDescription(
            sampleDescription, field_label=COLUMN_SAMPLE_DESCRIPTION)
        if not isValid:
            failed.append((COLUMN_SAMPLE_DESCRIPTION, errorMessage))

        isValid, errorMessage, gender_CV_value = sample_validator.validate_sampleGender(
            sampleGender, field_label=COLUMN_GENDER)
        if not isValid:
            failed.append((COLUMN_GENDER, errorMessage))

        isValid, errorMessage, role_CV_value = sample_validator.validate_sampleGroupType(
            sampleGroupType, field_label=COLUMN_GROUP_TYPE)
        if not isValid:
            failed.append((COLUMN_GROUP_TYPE, errorMessage))

        if sampleGroup:
            isValid, errorMessage = sample_validator.validate_sampleGroup(
                sampleGroup, field_label=COLUMN_GROUP)
            if not isValid:
                failed.append((COLUMN_GROUP, errorMessage))

        if cancerType:
            isValid, errorMessage, cancerType_CV_value = sample_validator.validate_cancerType(
                cancerType, field_label=COLUMN_CANCER_TYPE)
            if not isValid:
                failed.append((COLUMN_CANCER_TYPE, errorMessage))

        if cellularityPct:
            isValid, errorMessage = sample_validator.validate_cellularityPct(
                cellularityPct, field_label=COLUMN_CELLULARITY_PCT)
            if not isValid:
                failed.append((COLUMN_CELLULARITY_PCT, errorMessage))

        if pcrPlateRow:
            isValid, errorMessage, pcrPlateRow_internal_value = sample_validator.validate_pcrPlateRow(
                pcrPlateRow, field_label=COLUMN_PCR_PLATE_POSITION)
            if not isValid:
                failed.append((COLUMN_PCR_PLATE_POSITION, errorMessage))

        if nucleotideType:
            isValid, errorMessage, nucleotideType_internal_value = sample_validator.validate_nucleotideType(
                nucleotideType, field_label=COLUMN_NUCLEOTIDE_TYPE)
            if not isValid:
                failed.append((COLUMN_NUCLEOTIDE_TYPE, errorMessage))

        if biopsyDays:
            isValid, errorMessage = sample_validator.validate_sampleBiopsyDays(
                biopsyDays, field_label=COLUMN_BIOPSY_DAYS)
            if not isValid:
                failed.append((COLUMN_BIOPSY_DAYS, errorMessage))

        if cellNum:
            isValid, errorMessage = sample_validator.validate_sampleCellNum(
                cellNum, field_label=COLUMN_CELL_NUM)
            if not isValid:
                failed.append((COLUMN_CELL_NUM, errorMessage))

        if coupleId:
            isValid, errorMessage = sample_validator.validate_sampleCoupleId(
                coupleId, field_label=COLUMN_COUPLE_ID)
            if not isValid:
                failed.append((COLUMN_COUPLE_ID, errorMessage))

        if embryoId:
            isValid, errorMessage = sample_validator.validate_sampleEmbryoId(
                embryoId, field_label=COLUMN_EMBRYO_ID)
            if not isValid:
                failed.append((COLUMN_EMBRYO_ID, errorMessage))

        if sampleControlType:
            isValid, errorMessage, controlType_CV_value = sample_validator.validate_controlType(
                sampleControlType, field_label=COLUMN_CONTROLTYPE)
            if not isValid:
                failed.append((COLUMN_CONTROLTYPE, errorMessage))
        if population:
            isValid, errorMessage, _ = sample_validator.validate_population(
                population)
            if not isValid:
                failed.append((COLUMN_SAMPLE_POPULATION, errorMessage))
        if mouseStrains:
            isValid, errorMessage, _ = sample_validator.validate_mouseStrains(
                mouseStrains)
            if not isValid:
                failed.append((COLUMN_SAMPLE_MOUSE_STRAINS, errorMessage))
        if sampleSource:
            isValid, errorMessage, _ = sample_validator.validate_sampleSource(
                sampleSource)
            if not isValid:
                failed.append((COLUMN_SAMPLE_SOURCE, errorMessage))
        if panelPoolType:
            isValid, errorMessage, _ = sample_validator.validate_panelPoolType(
                panelPoolType)
            if not isValid:
                failed.append((COLUMN_PANEL_POOL_TYPE, errorMessage))
        if sampleCollectionDate:
            isValid, errorMessage = sample_validator.validate_sampleCollectionDate(
                sampleCollectionDate)
            if not isValid:
                failed.append((COLUMN_SAMPLE_COLLECTION_DATE, errorMessage))
        if sampleReceiptDate:
            isValid, errorMessage = sample_validator.validate_sampleReceiptDate(
                sampleReceiptDate, sampleCollectionDate)
            if not isValid:
                failed.append((COLUMN_SAMPLE_RECEIPT_DATE, errorMessage))

        # NEW VALIDATION FOR BARCODEKIT AND BARCODE_ID_STR
        isValid, errorMessage, item = sample_validator.validate_barcodekit_and_id_str(
            barcodeKit,
            barcodeAssignment,
            barcodeKit_label=COLUMN_BARCODE_KIT,
            barcode_id_str_label=COLUMN_BARCODE,
        )  # TODO: i18n
        if not isValid:
            if item == "barcodeKit":
                failed.append((COLUMN_BARCODE_KIT, errorMessage))
            else:
                failed.append((COLUMN_BARCODE, errorMessage))
        # if not isValid:
        #     failed.append((COLUMN_BARCODE, errorMessage))

        # validate user-defined custom attributes
        failed_userDefined = _validate_csv_user_defined_attributes(
            csvSampleDict, request)
        failed.extend(failed_userDefined)

        logger.debug(
            "import_sample_processor.validate_csv_sample() failed=%s" %
            (failed))

        return failed, isToSkipRow, isToAbort

    except Exception:
        logger.exception(format_exc())
        failed.append((
            "File Contents",
            " the CSV file does not seem to have all the columns. Click the Sample File Format button for an example. ",
        ))  # TODO: i18n
        logger.debug(
            "import_sample_processor.validate_csv_sample() failed=%s" %
            (failed))
        return failed, isToSkipRow, True
Example #6
0
def _create_sampleSetItem(csvSampleDict, request, user, sampleSet_ids):
    sampleDisplayedName = csvSampleDict.get(COLUMN_SAMPLE_NAME, '').strip()
    sampleExtId = csvSampleDict.get(COLUMN_SAMPLE_EXT_ID, '').strip()
    sampleGender = csvSampleDict.get(COLUMN_GENDER, '').strip()
    sampleGroupType = csvSampleDict.get(COLUMN_GROUP_TYPE, None)
    sampleGroup = csvSampleDict.get(COLUMN_GROUP, '0').strip()
    sampleDescription = csvSampleDict.get(COLUMN_SAMPLE_DESCRIPTION,
                                          '').strip()
    barcodeKit = csvSampleDict.get(COLUMN_BARCODE_KIT, '').strip()
    barcodeAssignment = csvSampleDict.get(COLUMN_BARCODE, '').strip()
    nucleotideType = csvSampleDict.get(COLUMN_NUCLEOTIDE_TYPE, "").strip()

    cancerType = csvSampleDict.get(COLUMN_CANCER_TYPE, "").strip()
    cellularityPct = csvSampleDict.get(COLUMN_CELLULARITY_PCT, None).strip()
    pcrPlateRow = csvSampleDict.get(COLUMN_PCR_PLATE_POSITION, "").strip()

    biopsyDays = csvSampleDict.get(COLUMN_BIOPSY_DAYS, "0").strip()
    coupleId = csvSampleDict.get(COLUMN_COUPLE_ID, None).strip()
    embryoId = csvSampleDict.get(COLUMN_EMBRYO_ID, "").strip()

    if not sampleGroup:
        sampleGroup = '0'

    isValid, errorMessage, nucleotideType_internal_value = sample_validator.validate_nucleotideType(
        nucleotideType)

    #validation has been done already, this is just to get the official value
    isValid, errorMessage, gender_CV_value = sample_validator.validate_sampleGender(
        sampleGender)
    isValid, errorMessage, role_CV_value = sample_validator.validate_sampleGroupType(
        sampleGroupType)

    isValid, errorMessage, cancerType_CV_value = sample_validator.validate_cancerType(
        cancerType)
    isValid, errorMessage, pcrPlateRow_internal_value = sample_validator.validate_pcrPlateRow(
        pcrPlateRow)

    sampleName = sampleDisplayedName.replace(' ', '_')
    sample_kwargs = {
        'displayedName': sampleDisplayedName,
        'status': 'created',
        'description': sampleDescription,
        'date': timezone.now()  ##datetime.datetime.now() 
    }

    sample, isCreated = Sample.objects.get_or_create(name=sampleName,
                                                     externalId=sampleExtId,
                                                     defaults=sample_kwargs)

    if isCreated:
        logger.debug(
            "import_sample_processor._create_sampleSetItem() new sample created for sample=%s; id=%d"
            % (sampleDisplayedName, sample.id))
    else:
        if (sample.description != sampleDescription):
            sample.description = sampleDescription
            sample.save()

            logger.debug(
                "import_sample_processor._create_sampleSetItem() just updated sample description for sample=%s; id=%d"
                % (sampleDisplayedName, sample.id))

    ##logger.debug("import_sample_processor._create_sampleSetItem() after get_or_create isCreated=%s; sample=%s; sample.id=%d" %(str(isCreated), sampleDisplayedName, sample.id))

    for sampleSetId in sampleSet_ids:
        logger.debug(
            "import_sample_processor._create_sampleSetItem() going to create sampleSetItem for sample=%s; sampleSetId=%s in sampleSet_ids=%s"
            % (sampleDisplayedName, str(sampleSetId), sampleSet_ids))

        currentDateTime = timezone.now()  ##datetime.datetime.now()

        dnabarcode = None
        if barcodeKit and barcodeAssignment:
            dnabarcode = models.dnaBarcode.objects.get(
                name__iexact=barcodeKit, id_str__iexact=barcodeAssignment)

        pcrPlateColumn = "1" if pcrPlateRow_internal_value else ""

        sampleSetItem_kwargs = {
            'gender': gender_CV_value,
            'relationshipRole': role_CV_value,
            'relationshipGroup': sampleGroup,
            'cancerType': cancerType_CV_value,
            'cellularityPct': cellularityPct if cellularityPct else None,
            'biopsyDays': int(biopsyDays) if biopsyDays else 0,
            "coupleId": coupleId,
            "embryoId": embryoId,
            'creator': user,
            'creationDate': currentDateTime,
            'lastModifiedUser': user,
            'lastModifiedDate': currentDateTime,
        }

        sampleSetItem, isCreated = SampleSetItem.objects.get_or_create(
            sample=sample,
            sampleSet_id=sampleSetId,
            dnabarcode=dnabarcode,
            nucleotideType=nucleotideType_internal_value,
            pcrPlateRow=pcrPlateRow_internal_value,
            pcrPlateColumn=pcrPlateColumn,
            defaults=sampleSetItem_kwargs)

        logger.debug(
            "import_sample_processor._create_sampleSetItem() after get_or_create isCreated=%s; sampleSetItem=%s; samplesetItem.id=%d"
            % (str(isCreated), sampleDisplayedName, sampleSetItem.id))

    ssi_sid = transaction.savepoint()

    return sample, sampleSetItem, ssi_sid
Example #7
0
def validate_csv_sample(csvSampleDict, request):
    """ 
    validate csv contents and convert user input to raw data to prepare for sample persistence
    returns: a collection of error messages if errors found and whether to skip the row
    """
    failed = []
    isToSkipRow = False
    isToAbort = False

    logger.debug(
        "ENTER import_sample_processor.validate_csv_sample() csvSampleDict=%s; "
        % (csvSampleDict))

    try:
        sampleDisplayedName = csvSampleDict.get(COLUMN_SAMPLE_NAME, '').strip()
        sampleExtId = csvSampleDict.get(COLUMN_SAMPLE_EXT_ID, '').strip()
        sampleGender = csvSampleDict.get(COLUMN_GENDER, '').strip()
        sampleGroupType = csvSampleDict.get(COLUMN_GROUP_TYPE, '').strip()
        sampleGroup = csvSampleDict.get(COLUMN_GROUP, '').strip()
        sampleDescription = csvSampleDict.get(COLUMN_SAMPLE_DESCRIPTION,
                                              '').strip()
        barcodeKit = csvSampleDict.get(COLUMN_BARCODE_KIT, '').strip()
        barcodeAssignment = csvSampleDict.get(COLUMN_BARCODE, '').strip()

        nucleotideType = csvSampleDict.get(COLUMN_NUCLEOTIDE_TYPE, "").strip()

        cancerType = csvSampleDict.get(COLUMN_CANCER_TYPE, "").strip()
        cellularityPct = csvSampleDict.get(COLUMN_CELLULARITY_PCT,
                                           None).strip()
        pcrPlateRow = csvSampleDict.get(COLUMN_PCR_PLATE_POSITION, "").strip()

        biopsyDays = csvSampleDict.get(COLUMN_BIOPSY_DAYS, "0").strip()
        coupleId = csvSampleDict.get(COLUMN_COUPLE_ID, "").strip()
        embryoId = csvSampleDict.get(COLUMN_EMBRYO_ID, "").strip()

        #skip blank line
        hasAtLeastOneValue = bool(
            [v for v in csvSampleDict.values() if v != ''])
        if not hasAtLeastOneValue:
            isToSkipRow = True
            return failed, isToSkipRow, isToAbort

        isValid, errorMessage = sample_validator.validate_sampleDisplayedName(
            sampleDisplayedName)
        if not isValid:
            failed.append((COLUMN_SAMPLE_NAME, errorMessage))

        isValid, errorMessage = sample_validator.validate_sampleExternalId(
            sampleExtId)
        if not isValid:
            failed.append((COLUMN_SAMPLE_EXT_ID, errorMessage))

        isValid, errorMessage = sample_validator.validate_sampleDescription(
            sampleDescription)
        if not isValid:
            failed.append((COLUMN_SAMPLE_DESCRIPTION, errorMessage))

        isValid, errorMessage, gender_CV_value = sample_validator.validate_sampleGender(
            sampleGender)
        if not isValid:
            failed.append((COLUMN_GENDER, errorMessage))

        isValid, errorMessage, role_CV_value = sample_validator.validate_sampleGroupType(
            sampleGroupType)
        if not isValid:
            failed.append((COLUMN_GROUP_TYPE, errorMessage))

        if sampleGroup:
            isValid, errorMessage = sample_validator.validate_sampleGroup(
                sampleGroup)
            if not isValid:
                failed.append((COLUMN_GROUP, errorMessage))

        if cancerType:
            isValid, errorMessage, cancerType_CV_value = sample_validator.validate_cancerType(
                cancerType)
            if not isValid:
                failed.append((COLUMN_CANCER_TYPE, errorMessage))

        if cellularityPct:
            isValid, errorMessage, value = sample_validator.validate_cellularityPct(
                cellularityPct)
            if not isValid:
                failed.append((COLUMN_CELLULARITY_PCT, errorMessage))

        if nucleotideType:
            isValid, errorMessage, nucleotideType_internal_value = sample_validator.validate_nucleotideType(
                nucleotideType)
            if not isValid:
                failed.append((COLUMN_NUCLEOTIDE_TYPE, errorMessage))

        if pcrPlateRow:
            isValid, errorMessage, pcrPlateRow_internal_value = sample_validator.validate_pcrPlateRow(
                pcrPlateRow)
            if not isValid:
                failed.append((COLUMN_PCR_PLATE_POSITION, errorMessage))

        if biopsyDays:
            isValid, errorMessage = sample_validator.validate_sampleBiopsyDays(
                biopsyDays)
            if not isValid:
                failed.append((COLUMN_BIOPSY_DAYS, errorMessage))

        if coupleId:
            isValid, errorMessage = sample_validator.validate_sampleCoupleId(
                coupleId)
            if not isValid:
                failed.append((COLUMN_COUPLE_ID, errorMessage))

        if embryoId:
            isValid, errorMessage = sample_validator.validate_sampleEmbryoId(
                embryoId)
            if not isValid:
                failed.append((COLUMN_EMBRYO_ID, errorMessage))

        ##NEW VALIDATION FOR BARCODEKIT AND BARCODE_ID_STR
        isValid, errorMessage, item = sample_validator.validate_barcodekit_and_id_str(
            barcodeKit, barcodeAssignment)
        if not isValid:
            if item == 'barcodeKit':
                failed.append((COLUMN_BARCODE_KIT, errorMessage))
            else:
                failed.append((COLUMN_BARCODE, errorMessage))
        # if not isValid:
        #     failed.append((COLUMN_BARCODE, errorMessage))

        #validate user-defined custom attributes
        failed_userDefined = _validate_csv_user_defined_attributes(
            csvSampleDict, request)
        failed.extend(failed_userDefined)

        logger.debug(
            "import_sample_processor.validate_csv_sample() failed=%s" %
            (failed))

        return failed, isToSkipRow, isToAbort

    except:
        logger.exception(format_exc())

        failed.append((
            "File Contents",
            " the CSV file does not seem to have all the columns. Click the Sample File Format button for an example. "
        ))
        isToAbort = True
        logger.debug(
            "import_sample_processor.validate_csv_sample() failed=%s" %
            (failed))
        return failed, isToSkipRow, isToAbort
Example #8
0
def _create_sampleSetItem(csvSampleDict, request, user, sampleSet_ids):
    sampleDisplayedName = csvSampleDict.get(COLUMN_SAMPLE_NAME, '').strip()
    sampleExtId = csvSampleDict.get(COLUMN_SAMPLE_EXT_ID, '').strip()
    sampleGender = csvSampleDict.get(COLUMN_GENDER, '').strip()
    sampleGroupType = csvSampleDict.get(COLUMN_GROUP_TYPE, None)
    sampleGroup = csvSampleDict.get(COLUMN_GROUP, '0').strip()
    sampleDescription = csvSampleDict.get(COLUMN_SAMPLE_DESCRIPTION, '').strip()
    barcodeKit = csvSampleDict.get(COLUMNS_BARCODE_KIT, '').strip()
    barcodeAssignment = csvSampleDict.get(COLUMN_BARCODE, '').strip()
    
    cancerType = csvSampleDict.get(COLUMN_CANCER_TYPE, "").strip()
    cellularityPct = csvSampleDict.get(COLUMN_CELLULARITY_PCT, None).strip()
    
    if not sampleGroup:
        sampleGroup = '0'

    #validation has been done already, this is just to get the official value
    isValid, errorMessage, gender_CV_value = sample_validator.validate_sampleGender(sampleGender)      
    isValid, errorMessage, role_CV_value = sample_validator.validate_sampleGroupType(sampleGroupType)

    isValid, errorMessage, cancerType_CV_value = sample_validator.validate_cancerType(cancerType)        
    
    sampleName = sampleDisplayedName.replace(' ', '_')
    sample_kwargs = {
                     'displayedName' : sampleDisplayedName,
                     'status' : 'created',
                     'description' : sampleDescription,
                     'date' : timezone.now()  ##datetime.datetime.now() 
                     }
    
    sample, isCreated = Sample.objects.get_or_create(name = sampleName, externalId = sampleExtId, defaults = sample_kwargs)

    if isCreated:
        logger.debug("import_sample_processor._create_sampleSetItem() new sample created for sample=%s; id=%d" %(sampleDisplayedName, sample.id))
    else:
        if (sample.description != sampleDescription):
            sample.description = sampleDescription
            sample.save()
            
            logger.debug("import_sample_processor._create_sampleSetItem() just updated sample description for sample=%s; id=%d" %(sampleDisplayedName, sample.id))
        
    logger.debug("import_sample_processor._create_sampleSetItem() after get_or_create isCreated=%s; sample=%s; sample.id=%d" %(str(isCreated), sampleDisplayedName, sample.id))


    for sampleSetId in sampleSet_ids: 
        logger.debug("import_sample_processor._create_sampleSetItem() going to create sampleSetItem for sample=%s; sampleSetId=%s in sampleSet_ids=%s" %(sampleDisplayedName, str(sampleSetId), sampleSet_ids))
        
        currentDateTime = timezone.now()  ##datetime.datetime.now() 

        dnabarcode = None
        if barcodeKit and barcodeAssignment:
            dnabarcode = models.dnaBarcode.objects.get(name__iexact=barcodeKit, id_str__iexact=barcodeAssignment)   
      
        sampleSetItem_kwargs = {
                                 'gender' : gender_CV_value, 
                                 'relationshipRole' : role_CV_value, 
                                 'relationshipGroup' : sampleGroup, 
                                 'cancerType' : cancerType_CV_value,
                                 'cellularityPct' : cellularityPct if cellularityPct else None,
                                 'creator' : user,
                                 'creationDate' : currentDateTime,
                                 'lastModifiedUser' : user,
                                 'lastModifiedDate' : currentDateTime,
                                 'dnabarcode' : dnabarcode                                
                             }
    
        sampleSetItem, isCreated = SampleSetItem.objects.get_or_create(sample = sample, 
                                                                       sampleSet_id = sampleSetId, 
                                                                       defaults = sampleSetItem_kwargs)

        logger.debug("import_sample_processor._create_sampleSetItem() after get_or_create isCreated=%s; sampleSetItem=%s; samplesetItem.id=%d" %(str(isCreated), sampleDisplayedName, sampleSetItem.id))            
            
    ssi_sid = transaction.savepoint()
    
    return sample, sampleSetItem, ssi_sid