def setUpTestData(cls):
# Create gating strategy
gating_strategy = GatingStrategy(strategy="Automatically Gated")
gating_strategy.save()
cls.gating_strategy = gating_strategy
# Create user needed for tests
user = User.objects.create_user(username="******",
email="*****@*****.**",
password="******")
user.save()
cls.user = user
# Get the base directory
cls.base_dir = os.path.dirname(os.path.realpath(__file__))
# Populate reference data table
fpath = os.path.join(cls.base_dir, "test_data",
"population_names_20200413.xlsx")
call_command("update_panel_parameter_reference_data", fpath)
# Upload sample patient data
fname = "test_panel_data_complete.csv"
uploaded_file = self._get_uploaded_file(fname)
clinical_sample_file = ClinicalSampleFile(
file_name=fname,
file_contents=uploaded_file,
user=self.user,
gating_strategy=self.gating_strategy,
)
validation_report = clinical_sample_file.validate()
upload_report = clinical_sample_file.upload()
def test_upload_results_with_derived_parameters(self):
"""If derived parameters present they should created dynamically and values uploaded"""
fname = "test_panel_data_with_derived_parameters.csv"
uploaded_file = self._get_uploaded_file(fname)
clinical_sample_file = ClinicalSampleFile(
file_name=fname,
file_contents=uploaded_file,
user=self.user,
gating_strategy=self.gating_strategy,
)
# Ensure unregistered derived parameters do not exist
unregistered_derived_parameters = [
"CD45p_03/Live_03 |freq",
"Time_03/Cells_03/Singlets1_03/Singlets2_03/Live_03/CD45p_03/T_03 | Count_back",
"Time_03/Cells_03/Singlets1_03/Singlets2_03/Live_03/CD45p_03/T_03/gd_03/gd_Vdx_03/Vdx_PD1p_03 | Count_back",
"Vdx_PD1p_03/gd_Vdx_03 |freq",
]
n_derived_parameters = Parameter.objects.filter(
gating_hierarchy__in=unregistered_derived_parameters
).count()
self.assertEqual(n_derived_parameters, 0)
upload_report = clinical_sample_file.upload()
# derived parameters should now be in Parameter table
n_derived_parameters = Parameter.objects.filter(
gating_hierarchy__in=unregistered_derived_parameters
).count()
self.assertEqual(n_derived_parameters, 4)
def test_fatal_error_on_missing_required_column(self):
"""A missing required column should give rise to a fatal error"""
file_name = "test_panel_data_missing_required_column.csv"
fpath = os.path.join(self.base_dir, "test_data", file_name)
with open(fpath, "rb") as infile:
uploaded_file = SimpleUploadedFile(
fpath, infile.read(), content_type="text/csv"
)
clinical_sample_file = ClinicalSampleFile(
file_name=file_name,
file_contents=uploaded_file,
user=self.user,
gating_strategy=self.gating_strategy,
)
validation_report = clinical_sample_file.validate()
length_of_validation_report = len(validation_report)
# There should be an entry in validation errors .
self.assertEquals(length_of_validation_report, 1)
# The first entry should have key: required_columns_missing
# type FATAL and value Clinical_sample
validation_entry = validation_report[0]
self.assertEquals(validation_entry.key, "required_columns_missing")
self.assertEquals(validation_entry.entry_type, "FATAL")
self.assertEquals(validation_entry.value, ["Clinical_sample"])
def test_info_on_derived_parameters(self):
"""If derived parameters present INFO should be given on validation"""
fname = "test_panel_data_with_derived_parameters.csv"
uploaded_file = self._get_uploaded_file(fname)
clinical_sample_file = ClinicalSampleFile(
file_name=fname,
file_contents=uploaded_file,
user=self.user,
gating_strategy=self.gating_strategy,
)
validation_report = clinical_sample_file.validate()
length_of_validation_report = len(validation_report)
validation_entry = validation_report[0]
self.assertEquals(
validation_entry.key,
"unregistered_derived_parameters - will be added during upload",
)
self.assertEquals(validation_entry.entry_type, "INFO")
unregistered_derived_parameters = [
"CD45p_03/Live_03 |freq",
"Time_03/Cells_03/Singlets1_03/Singlets2_03/Live_03/CD45p_03/T_03 | Count_back",
"Time_03/Cells_03/Singlets1_03/Singlets2_03/Live_03/CD45p_03/T_03/gd_03/gd_Vdx_03/Vdx_PD1p_03 | Count_back",
"Vdx_PD1p_03/gd_Vdx_03 |freq",
]
unregistered_derived_parameters.sort()
validation_entry.value.sort()
self.assertEquals(validation_entry.value, unregistered_derived_parameters)
def test_rows_with_invalid_fcs_file_name_not_loaded(self):
"""Rows with invalid clinical sample IDs should not be loaded"""
fname = "test_panel_data_fcs_file_name_nan.csv"
uploaded_file = self._get_uploaded_file(fname)
clinical_sample_file = ClinicalSampleFile(
file_name=fname,
file_contents=uploaded_file,
user=self.user,
gating_strategy=self.gating_strategy,
)
upload_report = clinical_sample_file.upload()
self.assertTrue(upload_report["rows_with_issues"], 2)
self.assertTrue(upload_report["rows_processed"], 3)
upload_issues = upload_report["validation"]
self.assertTrue(len(upload_issues), 2)
self.assertEquals(upload_issues[0].key, "row:0 field:filename")
self.assertEquals(
upload_issues[0].value,
"Value nan does not contain the sample ID (p005n01) - row not loaded",
)
self.assertEquals(upload_issues[1].key, "row:2 field:filename")
self.assertEquals(
upload_issues[1].value,
"Value nan does not contain the sample ID (p033n01) - row not loaded",
)
def test_complete_file_uploaded(self):
"""Test basic functionality works - all entries uploaded from sample file"""
fname = "test_panel_data_complete.csv"
uploaded_file = self._get_uploaded_file(fname)
clinical_sample_file = ClinicalSampleFile(
file_name=fname,
file_contents=uploaded_file,
user=self.user,
gating_strategy=self.gating_strategy,
)
validation_report = clinical_sample_file.validate()
upload_report = clinical_sample_file.upload()
self.assertTrue(len(upload_report), 0)
# Check we have expected entries
expected_results = self._get_expected_panel_results()
# Patients
self.assertTrue(Patient.objects.count(), 2)
for expected_result in expected_results:
patient_id = expected_result["patient_id"]
self.assertEqual(Patient.objects.filter(patient_id=patient_id).count(), 1)
# Samples
self.assertTrue(ProcessedSample.objects.count(), 2)
for expected_result in expected_results:
sample_id = expected_result["sample_id"]
self.assertEqual(
ProcessedSample.objects.filter(clinical_sample_id=sample_id).count(), 1
)
# Results
self.assertTrue(Result.objects.count(), 3)
for expected_result, result in zip(expected_results, Result.objects.all()):
self.assertEqual(
expected_result["sample_id"], result.processed_sample.clinical_sample_id
)
self.assertEqual(
expected_result["filename"], result.data_processing.fcs_file_name
)
self.assertEqual(expected_result["panel"].upper(), result.panel.name)
self.assertEqual(self.gating_strategy, result.gating_strategy)
self.assertEqual(fname, result.uploaded_file.name)
# Numeric values
# ToDo: Test actual values
self.assertEqual(NumericValue.objects.count(), 9)
# Date values
# ToDo: Test actual values
self.assertEqual(DateValue.objects.count(), 2)
# Text values
# ToDo: Test actual values
self.assertEqual(TextValue.objects.count(), 1)
def setUpTestData(cls):
# Create gating strategy
gating_strategy = GatingStrategy(strategy="Automatically Gated")
gating_strategy.save()
cls.gating_strategy = gating_strategy
# Create user needed for tests
user = User.objects.create_user(
username="******", email="*****@*****.**", password="******"
)
user.save()
cls.user = user
# Initialize the APIClient app
cls.client = Client()
# Get the base directory
cls.base_dir = os.path.dirname(os.path.realpath(__file__))
# Populate reference data table
fpath = os.path.join(
cls.base_dir, "test_data", "population_names_20200413.xlsx"
)
call_command("update_panel_parameter_reference_data", fpath)
# Upload test patients
fname = "test_patient_data.csv"
fpath = os.path.join(cls.base_dir, "test_data", fname)
with open(fpath, "rb") as infile:
uploaded_file = SimpleUploadedFile(
fpath, infile.read(), content_type="text/csv"
)
patient_file = PatientFile(
file_name=fname, file_contents=uploaded_file, user=cls.user,
)
upload_report = patient_file.upload()
# Upload sample patient data
fname = "test_panel_data_complete.csv"
uploaded_file = cls._get_uploaded_file(cls, fname)
clinical_sample_file = ClinicalSampleFile(
file_name=fname,
file_contents=uploaded_file,
user=cls.user,
gating_strategy=cls.gating_strategy,
)
validation_report = clinical_sample_file.validate()
upload_report = clinical_sample_file.upload()
def test_patient_id_created(self):
"""Test patient ID created during upload"""
fname = "test_panel_data_complete.csv"
uploaded_file = self._get_uploaded_file(fname)
clinical_sample_file = ClinicalSampleFile(
file_name=fname,
file_contents=uploaded_file,
user=self.user,
gating_strategy=self.gating_strategy,
)
upload_report = clinical_sample_file.upload()
self.assertTrue(len(upload_report), 0)
n_patients_created = Patient.objects.count()
self.assertEqual(n_patients_created, 2)
def test_all_expected_columns_present(self):
"""If static column missing or unexpected columns present, entry is made in validation errors"""
fpath = os.path.join(self.base_dir, "test_data", "test_panel_data_complete.csv")
with open(fpath, "rb") as infile:
uploaded_file = SimpleUploadedFile(
fpath, infile.read(), content_type="text/csv"
)
clinical_sample_file = ClinicalSampleFile(
file_name="test_panel_data_complete.csv",
file_contents=uploaded_file,
user=self.user,
gating_strategy=self.gating_strategy,
)
validation_report = clinical_sample_file.validate()
length_of_validation_report = len(validation_report)
# If no validation errors the validation report should be empty.
if length_of_validation_report > 0:
print("There were validation errors:")
print(validation_report)
self.assertEqual(length_of_validation_report, 0)
def upload(request):
if request.method == "POST":
file_type = None
if request.FILES.get("observationsFile"):
file_type = "observationsFile"
elif request.FILES.get("patientsFile"):
file_type = "patientsFile"
if file_type:
gating_strategy = GatingStrategy.objects.get_or_create(
strategy="manual")[0]
gating_strategy.save()
print(f"Gating strategy id = {gating_strategy.id}")
file_name = request.FILES[file_type].name
file_contents = request.FILES.get(file_type)
if file_type == "observationsFile":
uploaded_file = ClinicalSampleFile(
file_name,
file_contents,
user=request.user,
gating_strategy=gating_strategy,
)
else:
uploaded_file = PatientFile(file_name,
file_contents,
user=request.user)
validation_errors = uploaded_file.validate()
if validation_errors:
validation_report = {
"info": [
error for error in validation_errors
if error.entry_type == "INFO"
],
"warn": [
error for error in validation_errors
if error.entry_type == "WARN"
],
"error": [
error for error in validation_errors
if error.entry_type == "ERROR"
],
}
else:
validation_report = {}
upload_report = {}
if not uploaded_file.upload_file.valid_syntax:
print("Validation errors, aborting upload:")
print(validation_errors)
else:
print("trying upload")
try:
upload_report = uploaded_file.upload(dry_run=True)
upload_errors = {
"info": [
error for error in upload_report["validation"]
if error.entry_type == "INFO"
],
"warn": [
error for error in upload_report["validation"]
if error.entry_type == "WARN"
],
"error": [
error for error in upload_report["validation"]
if error.entry_type == "ERROR"
],
}
upload_report["validation"] = upload_errors
except Exception as e:
print("upload failed")
upload_report["status"] = "failed"
confirm_file_form = ConfirmFileForm(
initial={"file_id": uploaded_file.upload_file.id})
return render(
request,
"track/upload.html",
{
"uploaded": True,
"syntax_report": validation_report,
"model_report": upload_report,
"form": confirm_file_form,
},
)
elif ConfirmFileForm(request.POST).data.get("file_id"):
gating_strategy = GatingStrategy.objects.get_or_create(
strategy="manual")[0]
uploaded_file = UploadedFile.objects.get(
pk=ConfirmFileForm(request.POST).data.get("file_id"))
if uploaded_file.content_type == "PANEL_RESULTS":
uploaded_file = ClinicalSampleFile(
user=request.user,
uploaded_file=uploaded_file,
gating_strategy=gating_strategy,
)
else:
uploaded_file = PatientFile(user=request.user,
uploaded_file=uploaded_file)
uploaded_file.validate()
uploaded_file.upload()
return render(request, "track/upload.html",
{"upload_status": "success"})
return render(request, "track/upload.html")