def setUpTestData(cls): # Create gating strategy gating_strategy = GatingStrategy(strategy="Automatically Gated") gating_strategy.save() cls.gating_strategy = gating_strategy # Create user needed for tests user = User.objects.create_user(username="******", email="*****@*****.**", password="******") user.save() cls.user = user # Get the base directory cls.base_dir = os.path.dirname(os.path.realpath(__file__)) # Populate reference data table fpath = os.path.join(cls.base_dir, "test_data", "population_names_20200413.xlsx") call_command("update_panel_parameter_reference_data", fpath) # Upload sample patient data fname = "test_panel_data_complete.csv" uploaded_file = self._get_uploaded_file(fname) clinical_sample_file = ClinicalSampleFile( file_name=fname, file_contents=uploaded_file, user=self.user, gating_strategy=self.gating_strategy, ) validation_report = clinical_sample_file.validate() upload_report = clinical_sample_file.upload()
def test_fatal_error_on_missing_required_column(self): """A missing required column should give rise to a fatal error""" file_name = "test_panel_data_missing_required_column.csv" fpath = os.path.join(self.base_dir, "test_data", file_name) with open(fpath, "rb") as infile: uploaded_file = SimpleUploadedFile( fpath, infile.read(), content_type="text/csv" ) clinical_sample_file = ClinicalSampleFile( file_name=file_name, file_contents=uploaded_file, user=self.user, gating_strategy=self.gating_strategy, ) validation_report = clinical_sample_file.validate() length_of_validation_report = len(validation_report) # There should be an entry in validation errors . self.assertEquals(length_of_validation_report, 1) # The first entry should have key: required_columns_missing # type FATAL and value Clinical_sample validation_entry = validation_report[0] self.assertEquals(validation_entry.key, "required_columns_missing") self.assertEquals(validation_entry.entry_type, "FATAL") self.assertEquals(validation_entry.value, ["Clinical_sample"])
def test_info_on_derived_parameters(self): """If derived parameters present INFO should be given on validation""" fname = "test_panel_data_with_derived_parameters.csv" uploaded_file = self._get_uploaded_file(fname) clinical_sample_file = ClinicalSampleFile( file_name=fname, file_contents=uploaded_file, user=self.user, gating_strategy=self.gating_strategy, ) validation_report = clinical_sample_file.validate() length_of_validation_report = len(validation_report) validation_entry = validation_report[0] self.assertEquals( validation_entry.key, "unregistered_derived_parameters - will be added during upload", ) self.assertEquals(validation_entry.entry_type, "INFO") unregistered_derived_parameters = [ "CD45p_03/Live_03 |freq", "Time_03/Cells_03/Singlets1_03/Singlets2_03/Live_03/CD45p_03/T_03 | Count_back", "Time_03/Cells_03/Singlets1_03/Singlets2_03/Live_03/CD45p_03/T_03/gd_03/gd_Vdx_03/Vdx_PD1p_03 | Count_back", "Vdx_PD1p_03/gd_Vdx_03 |freq", ] unregistered_derived_parameters.sort() validation_entry.value.sort() self.assertEquals(validation_entry.value, unregistered_derived_parameters)
def test_complete_file_uploaded(self): """Test basic functionality works - all entries uploaded from sample file""" fname = "test_panel_data_complete.csv" uploaded_file = self._get_uploaded_file(fname) clinical_sample_file = ClinicalSampleFile( file_name=fname, file_contents=uploaded_file, user=self.user, gating_strategy=self.gating_strategy, ) validation_report = clinical_sample_file.validate() upload_report = clinical_sample_file.upload() self.assertTrue(len(upload_report), 0) # Check we have expected entries expected_results = self._get_expected_panel_results() # Patients self.assertTrue(Patient.objects.count(), 2) for expected_result in expected_results: patient_id = expected_result["patient_id"] self.assertEqual(Patient.objects.filter(patient_id=patient_id).count(), 1) # Samples self.assertTrue(ProcessedSample.objects.count(), 2) for expected_result in expected_results: sample_id = expected_result["sample_id"] self.assertEqual( ProcessedSample.objects.filter(clinical_sample_id=sample_id).count(), 1 ) # Results self.assertTrue(Result.objects.count(), 3) for expected_result, result in zip(expected_results, Result.objects.all()): self.assertEqual( expected_result["sample_id"], result.processed_sample.clinical_sample_id ) self.assertEqual( expected_result["filename"], result.data_processing.fcs_file_name ) self.assertEqual(expected_result["panel"].upper(), result.panel.name) self.assertEqual(self.gating_strategy, result.gating_strategy) self.assertEqual(fname, result.uploaded_file.name) # Numeric values # ToDo: Test actual values self.assertEqual(NumericValue.objects.count(), 9) # Date values # ToDo: Test actual values self.assertEqual(DateValue.objects.count(), 2) # Text values # ToDo: Test actual values self.assertEqual(TextValue.objects.count(), 1)
def setUpTestData(cls): # Create gating strategy gating_strategy = GatingStrategy(strategy="Automatically Gated") gating_strategy.save() cls.gating_strategy = gating_strategy # Create user needed for tests user = User.objects.create_user( username="******", email="*****@*****.**", password="******" ) user.save() cls.user = user # Initialize the APIClient app cls.client = Client() # Get the base directory cls.base_dir = os.path.dirname(os.path.realpath(__file__)) # Populate reference data table fpath = os.path.join( cls.base_dir, "test_data", "population_names_20200413.xlsx" ) call_command("update_panel_parameter_reference_data", fpath) # Upload test patients fname = "test_patient_data.csv" fpath = os.path.join(cls.base_dir, "test_data", fname) with open(fpath, "rb") as infile: uploaded_file = SimpleUploadedFile( fpath, infile.read(), content_type="text/csv" ) patient_file = PatientFile( file_name=fname, file_contents=uploaded_file, user=cls.user, ) upload_report = patient_file.upload() # Upload sample patient data fname = "test_panel_data_complete.csv" uploaded_file = cls._get_uploaded_file(cls, fname) clinical_sample_file = ClinicalSampleFile( file_name=fname, file_contents=uploaded_file, user=cls.user, gating_strategy=cls.gating_strategy, ) validation_report = clinical_sample_file.validate() upload_report = clinical_sample_file.upload()
def test_all_expected_columns_present(self): """If static column missing or unexpected columns present, entry is made in validation errors""" fpath = os.path.join(self.base_dir, "test_data", "test_panel_data_complete.csv") with open(fpath, "rb") as infile: uploaded_file = SimpleUploadedFile( fpath, infile.read(), content_type="text/csv" ) clinical_sample_file = ClinicalSampleFile( file_name="test_panel_data_complete.csv", file_contents=uploaded_file, user=self.user, gating_strategy=self.gating_strategy, ) validation_report = clinical_sample_file.validate() length_of_validation_report = len(validation_report) # If no validation errors the validation report should be empty. if length_of_validation_report > 0: print("There were validation errors:") print(validation_report) self.assertEqual(length_of_validation_report, 0)
def upload(request): if request.method == "POST": file_type = None if request.FILES.get("observationsFile"): file_type = "observationsFile" elif request.FILES.get("patientsFile"): file_type = "patientsFile" if file_type: gating_strategy = GatingStrategy.objects.get_or_create( strategy="manual")[0] gating_strategy.save() print(f"Gating strategy id = {gating_strategy.id}") file_name = request.FILES[file_type].name file_contents = request.FILES.get(file_type) if file_type == "observationsFile": uploaded_file = ClinicalSampleFile( file_name, file_contents, user=request.user, gating_strategy=gating_strategy, ) else: uploaded_file = PatientFile(file_name, file_contents, user=request.user) validation_errors = uploaded_file.validate() if validation_errors: validation_report = { "info": [ error for error in validation_errors if error.entry_type == "INFO" ], "warn": [ error for error in validation_errors if error.entry_type == "WARN" ], "error": [ error for error in validation_errors if error.entry_type == "ERROR" ], } else: validation_report = {} upload_report = {} if not uploaded_file.upload_file.valid_syntax: print("Validation errors, aborting upload:") print(validation_errors) else: print("trying upload") try: upload_report = uploaded_file.upload(dry_run=True) upload_errors = { "info": [ error for error in upload_report["validation"] if error.entry_type == "INFO" ], "warn": [ error for error in upload_report["validation"] if error.entry_type == "WARN" ], "error": [ error for error in upload_report["validation"] if error.entry_type == "ERROR" ], } upload_report["validation"] = upload_errors except Exception as e: print("upload failed") upload_report["status"] = "failed" confirm_file_form = ConfirmFileForm( initial={"file_id": uploaded_file.upload_file.id}) return render( request, "track/upload.html", { "uploaded": True, "syntax_report": validation_report, "model_report": upload_report, "form": confirm_file_form, }, ) elif ConfirmFileForm(request.POST).data.get("file_id"): gating_strategy = GatingStrategy.objects.get_or_create( strategy="manual")[0] uploaded_file = UploadedFile.objects.get( pk=ConfirmFileForm(request.POST).data.get("file_id")) if uploaded_file.content_type == "PANEL_RESULTS": uploaded_file = ClinicalSampleFile( user=request.user, uploaded_file=uploaded_file, gating_strategy=gating_strategy, ) else: uploaded_file = PatientFile(user=request.user, uploaded_file=uploaded_file) uploaded_file.validate() uploaded_file.upload() return render(request, "track/upload.html", {"upload_status": "success"}) return render(request, "track/upload.html")