Beispiel #1
0
    def setUpTestData(cls):
        # Create gating strategy
        gating_strategy = GatingStrategy(strategy="Automatically Gated")
        gating_strategy.save()
        cls.gating_strategy = gating_strategy

        # Create user needed for tests
        user = User.objects.create_user(username="******",
                                        email="*****@*****.**",
                                        password="******")
        user.save()
        cls.user = user

        # Get the base directory
        cls.base_dir = os.path.dirname(os.path.realpath(__file__))

        # Populate reference data table
        fpath = os.path.join(cls.base_dir, "test_data",
                             "population_names_20200413.xlsx")
        call_command("update_panel_parameter_reference_data", fpath)

        # Upload sample patient data
        fname = "test_panel_data_complete.csv"
        uploaded_file = self._get_uploaded_file(fname)
        clinical_sample_file = ClinicalSampleFile(
            file_name=fname,
            file_contents=uploaded_file,
            user=self.user,
            gating_strategy=self.gating_strategy,
        )

        validation_report = clinical_sample_file.validate()
        upload_report = clinical_sample_file.upload()
Beispiel #2
0
    def test_upload_results_with_derived_parameters(self):
        """If derived parameters present they should created dynamically and values uploaded"""

        fname = "test_panel_data_with_derived_parameters.csv"
        uploaded_file = self._get_uploaded_file(fname)
        clinical_sample_file = ClinicalSampleFile(
            file_name=fname,
            file_contents=uploaded_file,
            user=self.user,
            gating_strategy=self.gating_strategy,
        )

        # Ensure unregistered derived parameters do not exist
        unregistered_derived_parameters = [
            "CD45p_03/Live_03 |freq",
            "Time_03/Cells_03/Singlets1_03/Singlets2_03/Live_03/CD45p_03/T_03 | Count_back",
            "Time_03/Cells_03/Singlets1_03/Singlets2_03/Live_03/CD45p_03/T_03/gd_03/gd_Vdx_03/Vdx_PD1p_03 | Count_back",
            "Vdx_PD1p_03/gd_Vdx_03 |freq",
        ]
        n_derived_parameters = Parameter.objects.filter(
            gating_hierarchy__in=unregistered_derived_parameters
        ).count()
        self.assertEqual(n_derived_parameters, 0)

        upload_report = clinical_sample_file.upload()

        # derived parameters should now be in Parameter table
        n_derived_parameters = Parameter.objects.filter(
            gating_hierarchy__in=unregistered_derived_parameters
        ).count()
        self.assertEqual(n_derived_parameters, 4)
Beispiel #3
0
    def test_fatal_error_on_missing_required_column(self):
        """A missing required column should give rise to a fatal error"""

        file_name = "test_panel_data_missing_required_column.csv"
        fpath = os.path.join(self.base_dir, "test_data", file_name)
        with open(fpath, "rb") as infile:
            uploaded_file = SimpleUploadedFile(
                fpath, infile.read(), content_type="text/csv"
            )
        clinical_sample_file = ClinicalSampleFile(
            file_name=file_name,
            file_contents=uploaded_file,
            user=self.user,
            gating_strategy=self.gating_strategy,
        )

        validation_report = clinical_sample_file.validate()
        length_of_validation_report = len(validation_report)
        # There should be an entry in validation errors .
        self.assertEquals(length_of_validation_report, 1)

        # The first entry should have key: required_columns_missing
        # type FATAL and value Clinical_sample
        validation_entry = validation_report[0]
        self.assertEquals(validation_entry.key, "required_columns_missing")
        self.assertEquals(validation_entry.entry_type, "FATAL")
        self.assertEquals(validation_entry.value, ["Clinical_sample"])
Beispiel #4
0
 def test_info_on_derived_parameters(self):
     """If derived parameters present INFO should be given on validation"""
     fname = "test_panel_data_with_derived_parameters.csv"
     uploaded_file = self._get_uploaded_file(fname)
     clinical_sample_file = ClinicalSampleFile(
         file_name=fname,
         file_contents=uploaded_file,
         user=self.user,
         gating_strategy=self.gating_strategy,
     )
     validation_report = clinical_sample_file.validate()
     length_of_validation_report = len(validation_report)
     validation_entry = validation_report[0]
     self.assertEquals(
         validation_entry.key,
         "unregistered_derived_parameters - will be added during upload",
     )
     self.assertEquals(validation_entry.entry_type, "INFO")
     unregistered_derived_parameters = [
         "CD45p_03/Live_03 |freq",
         "Time_03/Cells_03/Singlets1_03/Singlets2_03/Live_03/CD45p_03/T_03 | Count_back",
         "Time_03/Cells_03/Singlets1_03/Singlets2_03/Live_03/CD45p_03/T_03/gd_03/gd_Vdx_03/Vdx_PD1p_03 | Count_back",
         "Vdx_PD1p_03/gd_Vdx_03 |freq",
     ]
     unregistered_derived_parameters.sort()
     validation_entry.value.sort()
     self.assertEquals(validation_entry.value, unregistered_derived_parameters)
Beispiel #5
0
    def test_rows_with_invalid_fcs_file_name_not_loaded(self):
        """Rows with invalid clinical sample IDs should not be loaded"""
        fname = "test_panel_data_fcs_file_name_nan.csv"
        uploaded_file = self._get_uploaded_file(fname)
        clinical_sample_file = ClinicalSampleFile(
            file_name=fname,
            file_contents=uploaded_file,
            user=self.user,
            gating_strategy=self.gating_strategy,
        )

        upload_report = clinical_sample_file.upload()
        self.assertTrue(upload_report["rows_with_issues"], 2)
        self.assertTrue(upload_report["rows_processed"], 3)

        upload_issues = upload_report["validation"]
        self.assertTrue(len(upload_issues), 2)
        self.assertEquals(upload_issues[0].key, "row:0 field:filename")
        self.assertEquals(
            upload_issues[0].value,
            "Value nan does not contain the sample ID (p005n01) - row not loaded",
        )
        self.assertEquals(upload_issues[1].key, "row:2 field:filename")
        self.assertEquals(
            upload_issues[1].value,
            "Value nan does not contain the sample ID (p033n01) - row not loaded",
        )
Beispiel #6
0
    def test_complete_file_uploaded(self):
        """Test basic functionality works - all entries uploaded from sample file"""

        fname = "test_panel_data_complete.csv"
        uploaded_file = self._get_uploaded_file(fname)
        clinical_sample_file = ClinicalSampleFile(
            file_name=fname,
            file_contents=uploaded_file,
            user=self.user,
            gating_strategy=self.gating_strategy,
        )

        validation_report = clinical_sample_file.validate()
        upload_report = clinical_sample_file.upload()
        self.assertTrue(len(upload_report), 0)

        # Check we have expected entries
        expected_results = self._get_expected_panel_results()

        # Patients
        self.assertTrue(Patient.objects.count(), 2)
        for expected_result in expected_results:
            patient_id = expected_result["patient_id"]
            self.assertEqual(Patient.objects.filter(patient_id=patient_id).count(), 1)

        # Samples
        self.assertTrue(ProcessedSample.objects.count(), 2)
        for expected_result in expected_results:
            sample_id = expected_result["sample_id"]
            self.assertEqual(
                ProcessedSample.objects.filter(clinical_sample_id=sample_id).count(), 1
            )

        # Results
        self.assertTrue(Result.objects.count(), 3)
        for expected_result, result in zip(expected_results, Result.objects.all()):
            self.assertEqual(
                expected_result["sample_id"], result.processed_sample.clinical_sample_id
            )
            self.assertEqual(
                expected_result["filename"], result.data_processing.fcs_file_name
            )
            self.assertEqual(expected_result["panel"].upper(), result.panel.name)
            self.assertEqual(self.gating_strategy, result.gating_strategy)
            self.assertEqual(fname, result.uploaded_file.name)

        # Numeric values
        # ToDo: Test actual values
        self.assertEqual(NumericValue.objects.count(), 9)

        # Date values
        # ToDo: Test actual values
        self.assertEqual(DateValue.objects.count(), 2)

        # Text values
        # ToDo: Test actual values
        self.assertEqual(TextValue.objects.count(), 1)
    def setUpTestData(cls):
        # Create gating strategy
        gating_strategy = GatingStrategy(strategy="Automatically Gated")
        gating_strategy.save()
        cls.gating_strategy = gating_strategy

        # Create user needed for tests
        user = User.objects.create_user(
            username="******", email="*****@*****.**", password="******"
        )
        user.save()
        cls.user = user

        # Initialize the APIClient app
        cls.client = Client()

        # Get the base directory
        cls.base_dir = os.path.dirname(os.path.realpath(__file__))

        # Populate reference data table
        fpath = os.path.join(
            cls.base_dir, "test_data", "population_names_20200413.xlsx"
        )
        call_command("update_panel_parameter_reference_data", fpath)

        # Upload test patients
        fname = "test_patient_data.csv"
        fpath = os.path.join(cls.base_dir, "test_data", fname)
        with open(fpath, "rb") as infile:
            uploaded_file = SimpleUploadedFile(
                fpath, infile.read(), content_type="text/csv"
            )
        patient_file = PatientFile(
            file_name=fname, file_contents=uploaded_file, user=cls.user,
        )
        upload_report = patient_file.upload()

        # Upload sample patient data
        fname = "test_panel_data_complete.csv"
        uploaded_file = cls._get_uploaded_file(cls, fname)
        clinical_sample_file = ClinicalSampleFile(
            file_name=fname,
            file_contents=uploaded_file,
            user=cls.user,
            gating_strategy=cls.gating_strategy,
        )

        validation_report = clinical_sample_file.validate()
        upload_report = clinical_sample_file.upload()
Beispiel #8
0
    def test_patient_id_created(self):
        """Test patient ID created during upload"""

        fname = "test_panel_data_complete.csv"
        uploaded_file = self._get_uploaded_file(fname)
        clinical_sample_file = ClinicalSampleFile(
            file_name=fname,
            file_contents=uploaded_file,
            user=self.user,
            gating_strategy=self.gating_strategy,
        )

        upload_report = clinical_sample_file.upload()
        self.assertTrue(len(upload_report), 0)

        n_patients_created = Patient.objects.count()
        self.assertEqual(n_patients_created, 2)
Beispiel #9
0
    def test_all_expected_columns_present(self):
        """If static column missing or unexpected columns present, entry is made in validation errors"""

        fpath = os.path.join(self.base_dir, "test_data", "test_panel_data_complete.csv")
        with open(fpath, "rb") as infile:
            uploaded_file = SimpleUploadedFile(
                fpath, infile.read(), content_type="text/csv"
            )
        clinical_sample_file = ClinicalSampleFile(
            file_name="test_panel_data_complete.csv",
            file_contents=uploaded_file,
            user=self.user,
            gating_strategy=self.gating_strategy,
        )

        validation_report = clinical_sample_file.validate()
        length_of_validation_report = len(validation_report)
        # If no validation errors the validation report should be empty.
        if length_of_validation_report > 0:
            print("There were validation errors:")
            print(validation_report)
        self.assertEqual(length_of_validation_report, 0)
Beispiel #10
0
def upload(request):
    if request.method == "POST":
        file_type = None
        if request.FILES.get("observationsFile"):
            file_type = "observationsFile"
        elif request.FILES.get("patientsFile"):
            file_type = "patientsFile"
        if file_type:
            gating_strategy = GatingStrategy.objects.get_or_create(
                strategy="manual")[0]
            gating_strategy.save()
            print(f"Gating strategy id = {gating_strategy.id}")
            file_name = request.FILES[file_type].name
            file_contents = request.FILES.get(file_type)
            if file_type == "observationsFile":
                uploaded_file = ClinicalSampleFile(
                    file_name,
                    file_contents,
                    user=request.user,
                    gating_strategy=gating_strategy,
                )
            else:
                uploaded_file = PatientFile(file_name,
                                            file_contents,
                                            user=request.user)
            validation_errors = uploaded_file.validate()
            if validation_errors:
                validation_report = {
                    "info": [
                        error for error in validation_errors
                        if error.entry_type == "INFO"
                    ],
                    "warn": [
                        error for error in validation_errors
                        if error.entry_type == "WARN"
                    ],
                    "error": [
                        error for error in validation_errors
                        if error.entry_type == "ERROR"
                    ],
                }
            else:
                validation_report = {}
            upload_report = {}
            if not uploaded_file.upload_file.valid_syntax:
                print("Validation errors, aborting upload:")
                print(validation_errors)
            else:
                print("trying upload")
                try:
                    upload_report = uploaded_file.upload(dry_run=True)
                    upload_errors = {
                        "info": [
                            error for error in upload_report["validation"]
                            if error.entry_type == "INFO"
                        ],
                        "warn": [
                            error for error in upload_report["validation"]
                            if error.entry_type == "WARN"
                        ],
                        "error": [
                            error for error in upload_report["validation"]
                            if error.entry_type == "ERROR"
                        ],
                    }
                    upload_report["validation"] = upload_errors
                except Exception as e:
                    print("upload failed")
                    upload_report["status"] = "failed"
            confirm_file_form = ConfirmFileForm(
                initial={"file_id": uploaded_file.upload_file.id})
            return render(
                request,
                "track/upload.html",
                {
                    "uploaded": True,
                    "syntax_report": validation_report,
                    "model_report": upload_report,
                    "form": confirm_file_form,
                },
            )
        elif ConfirmFileForm(request.POST).data.get("file_id"):
            gating_strategy = GatingStrategy.objects.get_or_create(
                strategy="manual")[0]
            uploaded_file = UploadedFile.objects.get(
                pk=ConfirmFileForm(request.POST).data.get("file_id"))
            if uploaded_file.content_type == "PANEL_RESULTS":
                uploaded_file = ClinicalSampleFile(
                    user=request.user,
                    uploaded_file=uploaded_file,
                    gating_strategy=gating_strategy,
                )
            else:
                uploaded_file = PatientFile(user=request.user,
                                            uploaded_file=uploaded_file)
            uploaded_file.validate()
            uploaded_file.upload()
            return render(request, "track/upload.html",
                          {"upload_status": "success"})
    return render(request, "track/upload.html")