def save_as(in_path: str, NewID: str, out_path: str) -> bool:
"""
Anonymize the DICOM to remove any identifiable information from a file and to a output file provided.
This operate at the memory level so should be quite a bit faster.
DICOM file check happens at the lowest DICOM_element level.
# NOTE! Expand here if you need to anonymize additional fields.
:param in_path:
:param NewID:
:param out_path:
:return:
"""
success, DICOM = DICOM_validate.file(in_path)
if not success:
return False
# Anonymize PatientID with the NewID provided.
success1, DICOM_updated = DICOM_elements.update_in_memory(
DICOM, "PatientID", NewID)
if not success1:
return False
# Anonymize PatientName with the NewID provided.
success2, DICOM_updated = DICOM_elements.update_in_memory(
DICOM_updated, "PatientName", NewID)
# Return after encuring both anonymization process are successful.
if success2:
DICOM_updated.save_as(out_path)
return True
else:
return False
def test_DICOM_update(self):
path = get_testdata_files("emri")
for file in path:
success, _ = DICOM_elements.update(file, "PatientBirthDate",
"19950101", file)
assert success
for file in path:
success, value = DICOM_elements.retrieve(file, "PatientBirthDate")
assert success
assert value == "19950101"
def test_element_retrieval():
# Get all files with JPEG in them.
file_names = recursive_list(r"C:\Users\Yang Ding\Desktop\test")
# files = get_testdata_files("[Jj][Pp][Ee][Gg]")
for file in file_names:
A, _ = DICOM_elements.retrieve(file,
"StudyDescription") # can be ''
B, _ = DICOM_elements.retrieve(
file, "PatientBirthDate") # can be 1995-12-19
C, _ = DICOM_elements.retrieve(file, "PatientSex") # can be M or F
assert A and B and C
def retrieve_study_protocol(files: List[str]) -> List[str]:
"""
From the list of files, find the names of all the possible studies descriptions.
#(0008,1030) Study Description e.g. FUNCTIONAL^Dr.Bohbot
:param files:
:return:
"""
protocols = []
from DICOMTransit.DICOM.elements import DICOM_elements
# Get DICOM files.
for file in files:
# Ensure it exists before attempting to retrieve it.
if os.path.exists(file):
success, study_protocol = DICOM_elements.retrieve(
file, "ProtocolName")
else:
logger.error(
f"Study protocol could not be retrieved from: {file}. Skipping this file!"
)
continue
# Only add if it is not already in the list (avoid duplicate, ensure unique entries
if (LORIS_validation.validate_projectID(study_protocol)
and study_protocol not in protocols):
protocols.append(study_protocol)
return protocols
def test_DICOM_retrieveMRN(self):
path = get_testdata_files("emri")
for file_name in path:
success = DICOM_anonymize.save(file_name, "1234567")
assert success
success, MRN = DICOM_elements.retrieve_MRN(file_name)
assert success
assert MRN == "1234567"
def test_DICOM_anonymizer(self):
file_names = get_testdata_files("emri")
for file_name in file_names:
success = DICOM_anonymize.save(file_name, "CNBP0010001")
assert success
for file_name in file_names:
success, value = DICOM_elements.retrieve(file_name, "PatientID")
assert success
assert value == "CNBP0010001"
def check_anonymization(files: list, anonymized_name) -> bool:
"""
A function to double check a list of files against the KNOWN anonymized value. This ensures that the anonymization actually gets carried out.
NOTE!!!!
This is the part where we have to ensure all the values are properly anonymized.
@todo: generalize this such that it will provide a list of fields and then anonymize them all from the database etc.
:param files: File must be the absolute path!
:param anonymized_name:
:return:
"""
from DICOMTransit.DICOM.elements import DICOM_elements
from DICOMTransit.DICOM.elements_batch import DICOM_elements_batch
from DICOMTransit.DICOM.validate import DICOM_validate
# Check every single file in the DICOM collections.
for file in tqdm(files, position=0):
success, DICOM = DICOM_validate.file(file)
if not success:
return False
properties = ["PatientID", "PatientName"]
properties_output = DICOM_elements_batch.retrieval(DICOM, properties)
success1, patient_id = DICOM_elements.retrieve_fast(DICOM, "PatientID")
success2, name = DICOM_elements.retrieve_fast(DICOM, "PatientName")
# bad retrieval.
if not success1 or not success2:
return False
# not properly anonymized patient ID
if not patient_id == anonymized_name:
return False
# not properly anonymized name.
if not name == anonymized_name:
return False
return True
def update_MRN(self):
"""
After passing the consistency check, update MRN record from one of the DICOM files.
:return:
"""
if self.check_validity():
from DICOMTransit.DICOM.elements import DICOM_elements
# dicom_files are already vetted, and all of them are consistent in terms of MRN, just load the MRN from first file.
success, self.MRN = DICOM_elements.retrieve_MRN(
self.dicom_files[0])
return success
else:
return False
def BatchDateCalculation(path):
file_list = recursive_list(path)
for file in file_list:
if DICOM_validate.file(file):
# Try to extract these information from the files.
success1, StudyDate = DICOM_elements.retrieve(file, "StudyDate")
success2, PatientBirthDate = DICOM_elements.retrieve(
file, "PatientBirthDate")
success3, age = DICOM_elements.compute_age(file)
# Skip this file if it is not successful.
if not success1 or not success2 or not success3:
continue
# Print, store and append the information acquired.
A = [file, StudyDate, PatientBirthDate, str(age)]
print(A)
output.append(A)
with open("output.csv", "w") as resultFile:
wr = csv.writer(resultFile, dialect="excel")
wr.writerow(output)
def retrieval(dicom_object: FileDataset,
DICOM_properties=List[str]) -> (bool, Optional[List[str]]):
"""
Retrieve a series of properties from an in memory DICOM object.
:param dicom_object:
:param DICOM_properties:
:return:
"""
list_properties = []
for DICOM_property in DICOM_properties:
success, retrieved_property = DICOM_elements.retrieve_fast(
dicom_object, DICOM_property)
if success:
list_properties.append(retrieved_property)
else:
return False, None
return True, list_properties
def update_sex(self):
"""
After passing the consistency check, update MRN record from one of the DICOM files.
:return:
"""
if self.check_validity():
from DICOMTransit.DICOM.elements import DICOM_elements
# dicom_files are already vetted, and all of them are consistent in terms of MRN, just load the sex from first file.
success, self.sex = DICOM_elements.retrieve_sex(
self.dicom_files[0])
if not success:
raise ValueError(
f"Could not retrieve sex from DICOM file {self.dicom_files[0]}"
)
return success
else:
return False
def update_scan_date(self):
"""
Retrieve the scan date from the DIOCM files and then update the DICOM archive.
:return:
"""
if self.check_validity():
from DICOMTransit.DICOM.elements import DICOM_elements
success, self.scan_date = DICOM_elements.retrieve_scan_date(
self.dicom_files[0])
if success:
return success
else:
from datetime import datetime
self.scan_date = datetime.strptime("19000101", "%Y%m%d")
return False
else:
return False
def retrieve_sUID(dicom_files: list, sample_rate: int = 10) -> List[str]:
"""
Check all dicom files to get a unique list of all possible series UIDs.
:param dicom_files:
:return:
"""
logger.debug(
"Commencing unique series UID retrieval across representative DICOM files provided. "
)
from DICOMTransit.DICOM.elements import DICOM_elements
# Randomly sample the list every 10 items as most DICOM scans have at least 10 DICOM files and this ensures performance.
short_list = dicom_files[0::sample_rate] # sample every 10 items.
list_unique_sUID = []
for file in tqdm(short_list, position=0):
success, UID = DICOM_elements.retrieve_seriesUID(file)
if UID not in list_unique_sUID:
list_unique_sUID.append(UID)
logger.debug(
"Finished compiling a representative samples of unique series UID across DICOM files provided."
)
return list_unique_sUID
def into_folder(input_folder, output_folder):
"""
This function sort a input folder with or without sub layers and automaticlly flatten everything into a folder before then MOVING them into protocol based folder sorted by acquisition series.
:param input_folder: Input_folder can be a root folder or flat.
:return:
"""
# Element to check: Series number.
from PythonUtils.PUFile import flatcopy
from PythonUtils.PUFolder import recursive_list
# Get files
file_list = recursive_list(input_folder)
if not os.path.isdir(output_folder):
os.mkdir(output_folder)
# copy them to a flat structure to the output folder.
flatcopy(file_list, output_folder, DICOM_validate.file)
# decompress them if necessary.
# oshelper_files.decompress_folder(output_folder)
# Get files list again.
file_list = recursive_list(output_folder)
exception_encountered = 0
logger.info("Sorting files into folders:")
# File here should be the FULL path.
for file in tqdm(file_list, position=0):
success1, SeriesNumber = DICOM_elements.retrieve(
file, "SeriesNumber")
success2, SeriesDescription = DICOM_elements.retrieve(
file, "SeriesDescription")
if not success1 or not success2:
logger.info(
f"Skipped file with no acquisition series information: {file}"
)
exception_encountered = exception_encountered + 1
continue
# Check MRI Series folder exists
DestinationFolder = str(SeriesNumber) + "_" + SeriesDescription
DestinationFolder = DestinationFolder.replace(" ", "_")
DestinationFolder = DestinationFolder.replace(":", "_")
DestinationFolder = DestinationFolder.replace(r"/", "_")
DestinationFolder = DestinationFolder.replace(r"\\", "_")
# Make destination folder if not exist.
os.chdir(output_folder)
if not os.path.exists(DestinationFolder):
os.mkdir(DestinationFolder)
# Get file name.
_, filename = os.path.split(file)
shutil.move(file, os.path.join(DestinationFolder, filename))
logger.info(f"Total error encountered: {str(exception_encountered)}")
def traversal(dir_path: str, consistency_check: bool = True):
"""
Some basic information of the participants must be consistent across the files, such as the SCAN DATE (assuming they are not scanning across MIDNIGHT POINT)
Birthday date, subject name, etc MUST BE CONSISTENT across a SINGLE subject's folder, RIGHT!
:param dir_path:
:returns: 0) if the path is valid, 2) list of ONLY the valid DICOM files.
"""
from DICOMTransit.DICOM.validate import DICOM_validate
# Reject bad input check
if not os.path.exists(dir_path) or not os.path.isdir(dir_path):
logger.error("Bad data folder path")
return False, None
# Get all possible files from the there.
files = recursive_list(dir_path)
# Used to record the first encountered patientID and name, and will check against subsequent folder for same matching information.
PatientID = ""
PatientName = ""
# List to store all validated DICOM files.
validated_DICOM_files = []
from DICOMTransit.DICOM.elements import DICOM_elements
logger.info(
"Traversing individual dicom file for validation information.")
list_unique_sUID = []
previous_sUID = None # a shorthand to bypass the list check.
# Check individual DICOM file for consistencies.
for file in tqdm(files, position=0):
# Skip current file if they are not DICOM files.
is_DICOM, dicom_obj = DICOM_validate.file(file)
if not is_DICOM:
logger.error(
f"Bad DICOM files detected: {file}. They are not returned in the validated list!"
)
continue
# The following section checks individual files and determine if all files have consistency name/patient etc.
# Useful for unanticipated ZIP files which can be contaminated.
# Not useful when dealing with ORTHANC output files.
if consistency_check:
# @todo: what if one of them is NONE?
# @todo: what if the date and other things are inconsistent?
# Record first instance of patient ID and patient name.
if PatientID == "" and PatientName == "":
Success, PatientID = DICOM_elements.retrieve_fast(
dicom_obj, "PatientID")
Success, PatientName = DICOM_elements.retrieve_fast(
dicom_obj, "PatientName")
# raise issue if not successful
if not Success:
logger.error(
"DICOM meta data retrieval failure EVEN for the first DICOM FILE?! Checking next one."
)
else:
name = PatientName.original_string.decode("latin_1")
logger.debug(
f"DICOM meta data retrieval success: {PatientID} {name}"
)
# Regardless of success of failure, must continue to process the next file.
continue
# Check consistencies across folders in terms of patient ID, NAME.
Success1, CurrentPatientID = DICOM_elements.retrieve_fast(
dicom_obj, "PatientID")
Success2, CurrentPatientName = DICOM_elements.retrieve_fast(
dicom_obj, "PatientName")
if not Success1 or not Success2:
logger.error(
"Could not retrieve fields for comparison. At least ONE DICOM file has inconsistent Patient ID/NAME field."
)
return False, None
if not (PatientID == CurrentPatientID) or not (
PatientName == CurrentPatientName):
logger.info(
"PatientID or Name mismatch from the dicom archive. .")
return False, None
success, UID = DICOM_elements.retrieve_fast(
dicom_obj, "SeriesInstanceUID")
# A quick UID check before the HEAVY list operation.
if not UID == previous_sUID and UID not in list_unique_sUID:
list_unique_sUID.append(UID)
validated_DICOM_files.append(file)
previous_sUID = UID
return True, validated_DICOM_files, list_unique_sUID
def test_DICOM_computerScanAge(self):
path = get_testdata_files("emri_small_RLE")[0]
success, Age = DICOM_elements.compute_age(path)
assert success
logger.info(Age.day)
