def testAppend(self):
"""MultiValue: Append of item converts it to required type..."""
multival = MultiValue(IS, [1, 5, 10])
multival.append('5')
self.assertTrue(isinstance(multival[-1], IS))
self.assertEqual(multival[-1], 5,
"Item set by append is not correct value")
def testExtend(self):
"""MultiValue: Extending a list converts all to required type"""
multival = MultiValue(IS, [1, 5, 10])
multival.extend(['7', 42])
self.assertTrue(isinstance(multival[-2], IS))
self.assertTrue(isinstance(multival[-1], IS))
self.assertEqual(multival[-2], 7, "Item set by extend not correct value")
def testExtend(self):
"""MultiValue: Extending a list converts all to required type"""
multival = MultiValue(IS, [1, 5, 10])
multival.extend(['7', 42])
self.assertTrue(isinstance(multival[-2], IS))
self.assertTrue(isinstance(multival[-1], IS))
self.assertEqual(multival[-2], 7, "Item set by extend not correct value")
def convert_PN(byte_string,
is_little_endian,
struct_format=None,
encoding=None):
"""Read and return string(s) as PersonName instance(s)"""
# XXX - We have to replicate MultiString functionality here because we can't decode
# easily here since that is performed in PersonNameUnicode
if byte_string and (byte_string.endswith(b' ')
or byte_string.endswith(b'\x00')):
byte_string = byte_string[:-1]
splitup = byte_string.split(b"\\")
if encoding and in_py3:
args = (encoding, )
else:
args = ()
# We would like to return string literals
if in_py3:
valtype = lambda x: PersonName(x, *args).decode()
else:
valtype = lambda x: PersonName(x, *args)
if len(splitup) == 1:
return valtype(splitup[0])
else:
return MultiValue(valtype, splitup)
def testSlice(self):
"""MultiValue: Setting slice converts items to required type."""
multival = MultiValue(IS, list(range(7)))
multival[2:7:2] = [4, 16, 36]
for val in multival:
self.assertTrue(isinstance(val, IS), "Slice IS value not correct type")
self.assertEqual(multival[4], 16, "Set by slice failed for item 4 of list")
def testSetIndex(self):
"""MultiValue: Setting list item converts it to required type"""
multival = MultiValue(IS, [1, 5, 10])
multival[1] = '7'
self.assertTrue(isinstance(multival[1], IS))
self.assertEqual(multival[1], 7,
"Item set by index is not correct value")
def convert_ATvalue(bytes, is_little_endian, struct_format=None):
"""Read and return AT (tag) data_element value(s)"""
length = len(bytes)
if length == 4:
return convert_tag(bytes, is_little_endian)
# length > 4
if length % 4 != 0:
logger.warn("Expected length to be multiple of 4 for VR 'AT', got length %d at file position 0x%x", length, fp.tell()-4)
return MultiValue(Tag,[convert_tag(bytes, is_little_endian, offset=x)
for x in range(0, length, 4)])
def testLimits(self):
"""MultiValue: Raise error if any item outside DICOM limits...."""
original_flag = dicom.config.enforce_valid_values
dicom.config.enforce_valid_values = True
self.assertRaises(
OverflowError, MultiValue, IS,
[1, -2**31 - 1
]) # Overflow error not raised for IS out of DICOM valid range
if python_version >= (2, 7): # will overflow anyway for python < 2.7
dicom.config.enforce_valid_values = False
i = MultiValue(
IS, [1, -2**31 - 1
]) # config enforce=False so should not raise error
dicom.config.enforce_valid_values = original_flag
def MultiString(val, valtype=str):
"""Split a bytestring by delimiters if there are any
val -- DICOM bytestring to split up
valtype -- default str, but can be e.g. UID to overwrite to a specific type
"""
# Remove trailing blank used to pad to even length
# 2005.05.25: also check for trailing 0, error made in PET files we are converting
if val and (val.endswith(' ') or val.endswith('\x00')):
val = val[:-1]
splitup = val.split("\\")
if len(splitup) == 1:
val = splitup[0]
return valtype(val) if val else val
else:
return MultiValue(valtype, splitup)
def MultiString(val, valtype=str):
"""Split a string by delimiters if there are any
val -- DICOM string to split up
valtype -- default str, but can be e.g. UID to overwrite to a specific type
"""
# Remove trailing blank used to pad to even length
# 2005.05.25: also check for trailing 0, error made in PET files we are converting
if val and (val.endswith(' ') or val.endswith('\x00')):
val = val[:-1]
# XXX --> simpler version python > 2.4 splitup = [valtype(x) if x else x for x in val.split("\\")]
splitup = []
for subval in val.split("\\"):
if subval:
splitup.append(valtype(subval))
else:
splitup.append(subval)
if len(splitup) == 1:
return splitup[0]
else:
return MultiValue(valtype, splitup)
def testMultiDS(self):
"""MultiValue: Multi-valued data elements can be created........"""
multival = MultiValue(DS, ['11.1', '22.2', '33.3'])
for val in multival:
self.assertTrue(isinstance(val, (DSfloat, DSdecimal)),
"Multi-value DS item not converted to DS")
def testAppend(self):
"""MultiValue: Append of item converts it to required type..."""
multival = MultiValue(IS, [1, 5, 10])
multival.append('5')
self.assertTrue(isinstance(multival[-1], IS))
self.assertEqual(multival[-1], 5, "Item set by append is not correct value")
def run(self, ident_dir, clean_dir):
# Get first date for tags set in relative_dates
date_adjust = None
audit_date_correct = None
if self.relative_dates is not None:
date_adjust = {
tag: first_date - datetime(1970, 1, 1)
for tag, first_date in self.get_first_date(
ident_dir, self.relative_dates).items()
}
for root, _, files in os.walk(ident_dir):
for filename in files:
if filename.startswith('.'):
continue
source_path = os.path.join(root, filename)
try:
ds = dicom.read_file(source_path)
except IOError:
logger.error('Error reading file %s' % source_path)
self.close_all()
return False
except InvalidDicomError: # DICOM formatting error
self.quarantine_file(source_path, ident_dir,
'Could not read DICOM file.')
continue
move, reason = self.check_quarantine(ds)
if move:
self.quarantine_file(source_path, ident_dir, reason)
continue
# Store adjusted dates for recovery
obfusc_dates = None
if self.relative_dates is not None:
obfusc_dates = {
tag: datetime.strptime(ds[tag].value, '%Y%m%d') -
date_adjust[tag]
for tag in self.relative_dates
}
# Keep CSA Headers
csa_headers = dict()
if self.keep_csa_headers and (0x29, 0x10) in ds:
csa_headers[(0x29, 0x10)] = ds[(0x29, 0x10)]
for offset in [0x10, 0x20]:
elno = (0x10 * 0x0100) + offset
csa_headers[(0x29, elno)] = ds[(0x29, elno)]
destination_dir = self.destination(source_path, clean_dir,
ident_dir)
if not os.path.exists(destination_dir):
os.makedirs(destination_dir)
try:
ds, study_pk = self.anonymize(ds)
except ValueError, e:
self.quarantine_file(
source_path, ident_dir,
'Error running anonymize function. There may be a '
'DICOM element value that does not match the specified'
' Value Representation (VR). Error was: %s' % e)
continue
# Recover relative dates
if self.relative_dates is not None:
for tag in self.relative_dates:
if audit_date_correct != study_pk and tag in AUDIT.keys(
):
self.audit.update(
ds[tag], obfusc_dates[tag].strftime('%Y%m%d'),
study_pk)
ds[tag].value = obfusc_dates[tag].strftime('%Y%m%d')
audit_date_correct = study_pk
# Restore CSA Header
if len(csa_headers) > 0:
for tag in csa_headers:
ds[tag] = csa_headers[tag]
# Set Patient Identity Removed to YES
t = Tag((0x12, 0x62))
ds[t] = DataElement(t, 'CS', 'YES')
# Set the De-identification method code sequence
method_ds = Dataset()
t = dicom.tag.Tag((0x8, 0x102))
if self.profile == 'clean':
method_ds[t] = DataElement(
t, 'DS', MultiValue(DS, ['113100', '113105']))
else:
method_ds[t] = DataElement(t, 'DS',
MultiValue(DS, ['113100']))
t = dicom.tag.Tag((0x12, 0x64))
ds[t] = DataElement(t, 'SQ', Sequence([method_ds]))
out_filename = ds[
SOP_INSTANCE_UID].value if self.rename else filename
clean_name = os.path.join(destination_dir, out_filename)
try:
ds.save_as(clean_name)
except IOError:
logger.error('Error writing file %s' % clean_name)
self.close_all()
return False
def rtDicomFromPreviousFolder(input_folder,
ctr_folder_names,
ctr_names,
base_img,
masks_np,
output_file_name,
prefix_name=''):
'''
:param input_folder: Input folder where the ORIGINAL DICOMS are stored
:param ctr_folder_names: Name of folders where to search for contours
:param ctr_names: Original contour names (this should have the same order as the contours in masks_np)
:param base_img: Itk base image to use for obtaining the positions
:param masks_np: This is the numpy 4D binary array with the masks for each contour (ctr, slice, w, h)
:param prefix_name: Additional prefix string to use in the new names of the contours
:return:
'''
print('Getting dicom DataSequence from previous data...')
# Gets the proper rt_folder to look for contours
cont_file = getLatestContourFolder(ctr_folder_names, input_folder)
lstFilesContour = io_com.get_dicom_files_in_folder(
join(input_folder, cont_file))
# ===================== Contours ==================
ds = dicom.read_file(lstFilesContour[0]) # Reads original dataset
final_ROIContourSequence_values = [
] # This should be an array of datasets and will be replaced from the original Filedataset
# Iterate over the ROIContourSequences (ak the contours) Type: Sequence
for new_idx_ctr, c_ctr_name in enumerate(ctr_names):
old_idx_ctr = [
i for i, x in enumerate(ds.StructureSetROISequence)
if x.ROIName == c_ctr_name
]
if len(old_idx_ctr) == 0: # If the ROI was not found
print(
F'ERROR the contour {c_ctr_name} was not found on previous RTStructure'
)
else:
# If the ROI is found, the we use it as a template and just modify the positions of the contours
old_idx_ctr = old_idx_ctr[0] # Get just the index
ds.StructureSetROISequence[
new_idx_ctr].ROIName == c_ctr_name # Use the new name for this index
ds.StructureSetROISequence[
new_idx_ctr].ROINumber == new_idx_ctr + 1 # Use the new name for this index
ds.Manufacturer = 'Deep Learning Biomarkers Group'
ds.SeriesDate = str(datetime.date.today().strftime('%Y%m%d'))
ds.SeriesDescription = 'NN Prediction'
ds.StructureSetName = 'NN Prediction'
print(F'\t Found {c_ctr_name} with idx {old_idx_ctr}!!!')
# Copy the ROIContourSequence of the OLD contour (VERY IMPORTANT STEP, we initalize with everything that was there before)
old_ROIContour_ds = ds.ROIContourSequence[old_idx_ctr]
ds.StructureSetROISequence[
old_idx_ctr].ROIName = F'{prefix_name}{c_ctr_name}'
cur_mask = masks_np[
new_idx_ctr, :, :, :] # Get the proper mask data
slices_w_data = np.unique(np.where(cur_mask > 0)[0])
cur_ContourSequence_values = []
# Use the first contour sequence as template (IMPORTANT)
old_ContourSequence_ds = old_ROIContour_ds.ContourSequence[0]
# Iterate slices with some contour inside. Transform the mask to contour and add it to the sequence
for idx_slice, slice in enumerate(slices_w_data):
# for idx_slice,slice in enumerate([82]):
# Get a contour from the 2D mask using OpenCV
ctr_pts = getContourMinPosFrom2DMask(cur_mask[slice, :, :])
# --- Use to visualize the contoured points for each slice and contour -----------
# import matplotlib.pyplot as plt
# plt.imshow(cur_mask[slice,:,:])
# plt.title(slice)
# for c_ctr in ctr_pts:
# plt.scatter(c_ctr[:,0,0],c_ctr[:,0,1],s=1)
# plt.show()
tot_ctrs = len(ctr_pts)
# Iterate over EACH contour and create a dataset for each of them
for idx_ctr in range(tot_ctrs):
cur_contourdata_ds = dataset(
) # This is the Contour data for this ContourSequence
# Copy the desired values from the OLD ContourSequence
cur_contourdata_ds.ContourGeometricType = old_ContourSequence_ds.ContourGeometricType
contourdata_values = []
tot_pts = ctr_pts[idx_ctr].shape[0]
this_slice_indx_pos = []
# Add the 'desired' format of (x,y,z) for each contour point
cur_ctr_pts = ctr_pts[idx_ctr]
for cur_pt in cur_ctr_pts:
this_slice_indx_pos.append(
[int(cur_pt[0][0]),
int(cur_pt[0][1]),
int(slice)])
# We add again the first point of each contour at the end (trying to 'close' the contour)
this_slice_indx_pos.append([
int(cur_ctr_pts[0][0][0]),
int(cur_ctr_pts[0][0][1]),
int(slice)
])
# Transform each point into a physical position
for c_pos in this_slice_indx_pos:
phys_pos = base_img.TransformIndexToPhysicalPoint(
c_pos)
contourdata_values.append(phys_pos[0])
contourdata_values.append(phys_pos[1])
contourdata_values.append(phys_pos[2])
# Copy the list of physical positions in the variable ContourData of the dataset
cur_contourdata_ds.ContourData = MultiValue(
dicom.valuerep.DSfloat, contourdata_values)
cur_contourdata_ds.NumberOfContourPoints = tot_pts
# Append this dataset into the list of slices with date
cur_ContourSequence_values.append(cur_contourdata_ds)
old_ROIContour_ds.ContourSequence = Sequence(
cur_ContourSequence_values
) # Replace the desired sequence with the new values
final_ROIContourSequence_values.append(
old_ROIContour_ds) # Append it to the ROIContourSquence values
# Replace the original Sequence of ROIContours with the new ones. These were initialized with the previous values
# and only the Contour Data was replaced
new_ROIContourSequence = Sequence(final_ROIContourSequence_values)
ds.ROIContourSequence = new_ROIContourSequence # Replace old ROIContourSequence with new one
dicom.dcmwrite(output_file_name, ds)
def _storeIdentity(self, dcmFile):
"""
"""
# Collect attributes I want to keep and encrypt
protectedAttributes = []
# Frame Of Reference UID
if "FrameOfReferenceUID" in dcmFile:
ds1 = Dataset()
ds1[0x0020, 0x0052] = dcmFile[0x0020, 0x0052]
protectedAttributes.append(ds1)
# Patient ID
if "PatientID" in dcmFile:
ds2 = Dataset()
ds2[0x0010, 0x0020] = dcmFile[0x0010, 0x0020]
protectedAttributes.append(ds2)
# Patient name
if "PatientName" in dcmFile:
ds3 = Dataset()
ds3[0x0010, 0x0010] = dcmFile[0x0010, 0x0010]
protectedAttributes.append(ds3)
# Patient birth date
if "PatientBirthDate" in dcmFile:
ds4 = Dataset()
ds4[0x0010, 0x0030] = dcmFile[0x0010, 0x0030]
protectedAttributes.append(ds4)
# SOP Instance UID
if "SOPInstanceUID" in dcmFile:
ds5 = Dataset()
ds5[0x0008, 0x0018] = dcmFile[0x0008, 0x0018]
protectedAttributes.append(ds5)
# StudyInstance UID
if "StudyInstanceUID" in dcmFile:
ds6 = Dataset()
ds6[0x0020, 0x000D] = dcmFile[0x0020, 0x000D]
protectedAttributes.append(ds6)
# Instance of Encrypted Attributes Data Set
encryptedAttributesDs = Dataset()
# Set the Modified Attributes Sequence (0400,0550) to
# the Attributes to be protected
t = dicom.tag.Tag((0x400, 0x550))
encryptedAttributesDs[t] = dicom.dataelem.DataElement(
t, "SQ", Sequence(protectedAttributes))
# Serialize these original DICOM data to string
encryptedDicomAttributes = pickle.dumps(encryptedAttributesDs)
# Encrypt
encryptedData = self._svcCrypto.encrypt(encryptedDicomAttributes)
# Encrypted Attributes Sequence item with two attributes
item = Dataset()
# Set the attribute Encrypted Content Transfer Syntax UID (0400,0510) to
# the UID of the Transfer Syntax used to encode the instance of the Encrypted Attributes Data Set
t = dicom.tag.Tag((0x400, 0x510))
item[t] = dicom.dataelem.DataElement(
t, "UI", dcmFile.file_meta[0x0002, 0x0010].value)
# Set the atribute Encrypted Content (0400,0520) to
# the data resulting from the encryption of the Encrypted Attributes Data Set instance
t = dicom.tag.Tag((0x400, 0x520))
item[t] = dicom.dataelem.DataElement(t, "OB", encryptedData)
# Set the attribute Encrypted Attributes Sequence (0400,0500)
# each item consists of two attributes ( (0400,0510); (0400,0520) )
t = dicom.tag.Tag((0x400, 0x500))
dcmFile[t] = dicom.dataelem.DataElement(t, "SQ", Sequence([item]))
# Set the attribute Patient Identity Removed (0012,0062) to YES
t = dicom.tag.Tag((0x12, 0x62))
dcmFile[t] = dicom.dataelem.DataElement(t, "CS", "YES")
# Codes of corresponding profiles and options as a dataset
profilesOptionsDs = Dataset()
# De-identification Method Coding Scheme Designator (0008,0102)
t = dicom.tag.Tag((0x8, 0x102))
profilesOptionsDs[t] = dicom.dataelem.DataElement(
t, "DS",
MultiValue(dicom.valuerep.DS,
self._deidentConfig.GetAppliedMethodCodes()))
# Set the attribute De-identification method code sequence (0012,0064)
# to the created dataset of codes for profiles and options
t = dicom.tag.Tag((0x12, 0x64))
dcmFile[t] = dicom.dataelem.DataElement(t, "SQ",
Sequence([profilesOptionsDs]))
