def test_fieldset_remove_private(self):
"""
%fields field_set2_private
FIELD 00090010
FIELD PatientID
%header
REMOVE fields:field_set2_private
"""
print("Test private tag fieldset")
dicom_file = get_file(self.dataset)
actions = [{"action": "REMOVE", "field": "fields:field_set2_private"}]
fields = OrderedDict()
fields["field_set2_private"] = [
{
"field": "00090010",
"action": "FIELD"
},
{
"field": "PatientID",
"action": "FIELD"
},
]
recipe = create_recipe(actions, fields)
parser = DicomParser(dicom_file, recipe=recipe)
parser.parse()
self.assertTrue("(0009, 0010)" in parser.lookup["field_set2_private"])
self.assertTrue("(0010, 0020)" in parser.lookup["field_set2_private"])
with self.assertRaises(KeyError):
check1 = parser.dicom["00090010"].value
with self.assertRaises(KeyError):
check2 = parser.dicom["PatientID"].value
def get_identifiers(
dicom_files, force=True, config=None, strip_sequences=False, remove_private=False
):
""" extract all identifiers from a dicom image.
This function returns a lookup by file name, where each value indexed
includes a dictionary of nested fields (indexed by nested tag).
Parameters
==========
dicom_files: the dicom file(s) to extract from
force: force reading the file (default True)
config: if None, uses default in provided module folder
strip_sequences: if True, remove all sequences
remove_private: remove private tags
"""
if config is None:
config = "%s/config.json" % here
if not os.path.exists(config):
bot.error("Cannot find config %s, exiting" % (config))
config = read_json(config, ordered_dict=True)["get"]
if not isinstance(dicom_files, list):
dicom_files = [dicom_files]
bot.debug("Extracting identifiers for %s dicom" % len(dicom_files))
lookup = dict()
# Parse each dicom file
for dicom_file in dicom_files:
parser = DicomParser(dicom_file, force=force)
lookup[parser.dicom_file] = parser.get_fields()
return lookup
def test_fieldset_remove(self):
""" RECIPE
%fields field_set1
FIELD Manufacturer
FIELD contains:Time
%header
REMOVE fields:field_set1
"""
print("Test public tag fieldset")
dicom_file = get_file(self.dataset)
actions = [{"action": "REMOVE", "field": "fields:field_set1"}]
fields = OrderedDict()
fields["field_set1"] = [
{
"field": "Manufacturer",
"action": "FIELD"
},
{
"field": "contains:Collimation",
"action": "FIELD"
},
]
recipe = create_recipe(actions, fields)
# Method 1: Use DicomParser
parser = DicomParser(dicom_file, recipe=recipe)
number_fields = len(parser.dicom) # 160
parser.parse()
# The number of fields to be removed
to_remove = len(parser.lookup["field_set1"])
expected_number = number_fields - to_remove
# {'field_set1': {'(0008, 0070)': (0008, 0070) Manufacturer LO: 'SIEMENS' [Manufacturer],
# '(0018, 9306)': (0018, 9306) Single Collimation Width FD: 1.2 [SingleCollimationWidth],
# '(0018, 9307)': (0018, 9307) Total Collimation Width FD: 14.399999999999999 [TotalCollimationWidth]}}
# Method 1: use replace_identifiers
result = replace_identifiers(
dicom_files=dicom_file,
deid=recipe,
save=False,
remove_private=False,
strip_sequences=False,
)
self.assertEqual(1, len(result))
print(len(result[0]))
self.assertEqual(expected_number, len(result[0]))
with self.assertRaises(KeyError):
check1 = result[0]["Manufacturer"].value
with self.assertRaises(KeyError):
check2 = result[0]["TotalCollimationWidth"].value
with self.assertRaises(KeyError):
check3 = result[0]["SingleCollimationWidth"].value
def test_remove(self):
""" RECIPE RULE
REMOVE InstitutionName
REMOVE 00190010
"""
print("Test remove of public and private tags")
dicom_file = get_file(self.dataset)
field1name = "InstitutionName"
field2name = "00190010"
actions = [
{
"action": "REMOVE",
"field": field1name
},
{
"action": "REMOVE",
"field": field2name
},
]
recipe = create_recipe(actions)
dicom = read_file(dicom_file)
# Create a DicomParser to easily find fields
parser = DicomParser(dicom_file)
parser.parse()
# The first in the list is the highest level
field1 = list(parser.find_by_name(field1name).values())[0]
field2 = list(parser.find_by_name(field2name).values())[0]
self.assertIsNotNone(field1.element.value)
self.assertIsNotNone(field2.element.value)
result = replace_identifiers(
dicom_files=dicom_file,
deid=recipe,
save=False,
remove_private=False,
strip_sequences=False,
)
# Create a DicomParser to easily find fields
parser = DicomParser(result[0])
parser.parse()
# Removed means we don't find them
assert not parser.find_by_name(field1name)
assert not parser.find_by_name(field2name)
self.assertEqual(1, len(result))
with self.assertRaises(KeyError):
check1 = result[0][field1name].value
with self.assertRaises(KeyError):
check2 = result[0][field2name].value
def replace_identifiers(
dicom_files,
ids=None,
deid=None,
save=False,
overwrite=False,
output_folder=None,
force=True,
config=None,
strip_sequences=False,
remove_private=False,
):
"""replace identifiers using pydicom, can be slow when writing
and saving new files. If you want to replace sequences, they need
to be extracted with get_identifiers and expand_sequences to True.
"""
if not isinstance(dicom_files, list):
dicom_files = [dicom_files]
# Warn the user that we use the default deid recipe
if not deid:
bot.warning("No deid specification provided, will use defaults.")
# ids (a lookup) is not required
ids = ids or {}
# Parse through dicom files, update headers, and save
updated_files = []
for dicom_file in dicom_files:
parser = DicomParser(dicom_file, force=force, config=config, recipe=deid)
# If a custom lookup was provided, update the parser
if parser.dicom_file in ids:
parser.lookup.update(ids[parser.dicom_file])
parser.parse(strip_sequences=strip_sequences, remove_private=remove_private)
# Save to file, otherwise return updated objects
if save is True:
ds = save_dicom(
dicom=parser.dicom,
dicom_file=parser.dicom_file,
output_folder=output_folder,
overwrite=overwrite,
)
updated_files.append(ds)
else:
updated_files.append(parser.dicom)
return updated_files
def test_valueset_remove_one_empty(self):
"""
Testing to ensure correct actions are taken when a defined valueset contains a field that has an empty value. Since the
ConversionType flag contains "No Value", in the test below, value_set1 will only have the value from Manufacturer and should
only identify the fields which contain "SIEMENS".
%values value_set1
FIELD ConversionType
FIELD Manufacturer
%header
REMOVE values:value_set1
"""
import pydicom
print("Test one empty value valueset")
dicom_file = get_file(self.dataset)
original_dataset = pydicom.dcmread(dicom_file)
actions = [{"action": "REMOVE", "field": "values:value_set1"}]
values = OrderedDict()
values["value_set1"] = [
{
"field": "ConversionType",
"action": "FIELD"
},
{
"field": "Manufacturer",
"action": "FIELD"
},
]
recipe = create_recipe(actions, values=values)
# Check that values we want are present using DicomParser
parser = DicomParser(dicom_file, recipe=recipe)
parser.parse()
self.assertEqual(len(parser.lookup["value_set1"]), 1)
self.assertTrue("SIEMENS" in parser.lookup["value_set1"])
# Perform action
result = replace_identifiers(
dicom_files=dicom_file,
deid=recipe,
save=False,
remove_private=False,
strip_sequences=False,
)
self.assertEqual(1, len(result))
self.assertNotEqual(len(original_dataset), len(result[0]))
with self.assertRaises(KeyError):
check1 = result[0]["00090010"].value
with self.assertRaises(KeyError):
check2 = result[0]["Manufacturer"].value
def test_replace_with_constant(self):
""" RECIPE RULE
REPLACE AccessionNumber 987654321
REPLACE 00190010 NEWVALUE!
"""
print("Test replace tags with constant values")
dicom_file = get_file(self.dataset)
newfield1 = "AccessionNumber"
newvalue1 = "987654321"
newfield2 = "00190010"
newvalue2 = "NEWVALUE!"
actions = [
{
"action": "REPLACE",
"field": newfield1,
"value": newvalue1
},
{
"action": "REPLACE",
"field": newfield2,
"value": newvalue2
},
]
recipe = create_recipe(actions)
# Create a DicomParser to easily find fields
parser = DicomParser(dicom_file)
parser.parse()
# The first in the list is the highest level
field1 = list(parser.find_by_name(newfield1).values())[0]
field2 = list(parser.find_by_name(newfield2).values())[0]
self.assertNotEqual(newvalue1, field1.element.value)
self.assertNotEqual(newvalue2, field2.element.value)
result = replace_identifiers(
dicom_files=dicom_file,
deid=recipe,
save=False,
remove_private=False,
strip_sequences=False,
)
self.assertEqual(1, len(result))
self.assertEqual(newvalue1, result[0][newfield1].value)
self.assertEqual(newvalue2, result[0][newfield2].value)
def test_valueset_remove(self):
"""
%values value_set1
FIELD contains:Manufacturer
SPLIT contains:Physician by="^";minlength=3
%header REMOVE values:value_set1
"""
print("Test public tag valueset")
dicom_file = get_file(self.dataset)
actions = [{"action": "REMOVE", "field": "values:value_set1"}]
values = OrderedDict()
values["value_set1"] = [
{
"field": "contains:Manufacturer",
"action": "FIELD"
},
{
"value": 'by="^";minlength=3',
"field": "contains:Physician",
"action": "SPLIT",
},
]
recipe = create_recipe(actions, values=values)
# Check that values we want are present using DicomParser
parser = DicomParser(dicom_file, recipe=recipe)
parser.parse()
self.assertTrue("SIEMENS" in parser.lookup["value_set1"])
self.assertTrue("HIBBARD" in parser.lookup["value_set1"])
# Perform action
result = replace_identifiers(
dicom_files=dicom_file,
deid=recipe,
save=False,
remove_private=False,
strip_sequences=False,
)
self.assertEqual(1, len(result))
with self.assertRaises(KeyError):
check1 = result[0]["00090010"].value
with self.assertRaises(KeyError):
check2 = result[0]["Manufacturer"].value
with self.assertRaises(KeyError):
check3 = result[0]["PhysiciansOfRecord"].value
def test_valueset_empty_remove(self):
"""
Testing to ensure correct actions are taken when a defined valueset contains no data (the field identified has an empty value). Since the
ConversionType flag contains "No Value", in the test below, value_set1 will be empty and as a result this combination of rules should have no
impact on the header. The input header should be identical to the output header.
%values value_set1
FIELD ConversionType
%header
REMOVE values:value_set1
"""
import pydicom
print("Test empty value valueset")
dicom_file = get_file(self.dataset)
original_dataset = pydicom.dcmread(dicom_file)
actions = [{"action": "REMOVE", "field": "values:value_set1"}]
values = OrderedDict()
values["value_set1"] = [
{
"field": "ConversionType",
"action": "FIELD"
},
]
recipe = create_recipe(actions, values=values)
# Check that values we want are present using DicomParser
parser = DicomParser(dicom_file, recipe=recipe)
parser.parse()
self.assertEqual(len(parser.lookup["value_set1"]), 0)
# Perform action
result = replace_identifiers(
dicom_files=dicom_file,
deid=recipe,
save=False,
remove_private=False,
strip_sequences=False,
)
self.assertEqual(1, len(result))
self.assertEqual(len(original_dataset), len(result[0]))
def test_valueset_private(self):
"""
%values value_set2_private
FIELD 00311020
SPLIT 00090010 by=" ";minlength=4
%header
REMOVE values:value_set2_private
"""
print("Test private tag valueset")
dicom_file = get_file(self.dataset)
actions = [{"action": "REMOVE", "field": "values:value_set2_private"}]
values = OrderedDict()
values["value_set2_private"] = [
{
"field": "00311020",
"action": "FIELD"
},
{
"value": 'by=" ";minlength=4',
"field": "00090010",
"action": "SPLIT"
},
]
recipe = create_recipe(actions, values=values)
parser = DicomParser(dicom_file, recipe=recipe)
parser.parse()
for entry in ["SIEMENS", "M1212121", "DUMMY"]:
assert entry in parser.lookup["value_set2_private"]
with self.assertRaises(KeyError):
check1 = parser.dicom["OtherPatientIDs"].value
with self.assertRaises(KeyError):
check2 = parser.dicom["Manufacturer"].value
with self.assertRaises(KeyError):
check3 = parser.dicom["00190010"].value
def test_remove_all_func(self):
"""
%header
REMOVE ALL func:contains_hibbard
"""
print("Test tag removal by")
dicom_file = get_file(self.dataset)
def contains_hibbard(dicom, value, field, item):
from pydicom.tag import Tag
tag = Tag(field.element.tag)
if tag in dicom:
currentvalue = str(dicom.get(tag).value).lower()
if "hibbard" in currentvalue:
return True
return False
actions = [{
"action": "REMOVE",
"field": "ALL",
"value": "func:contains_hibbard"
}]
recipe = create_recipe(actions)
# Create a parser, define function for it
parser = DicomParser(dicom_file, recipe=recipe)
parser.define("contains_hibbard", contains_hibbard)
parser.parse()
self.assertEqual(156, len(parser.dicom))
with self.assertRaises(KeyError):
check1 = parser.dicom["ReferringPhysicianName"].value
with self.assertRaises(KeyError):
check2 = parser.dicom["PhysiciansOfRecord"].value
with self.assertRaises(KeyError):
check3 = parser.dicom["RequestingPhysician"].value
with self.assertRaises(KeyError):
check4 = parser.dicom["00331019"].value
def test_extract_groups(self):
print("Test deid.dicom.groups extract_values_list")
from deid.dicom.groups import extract_values_list, extract_fields_list
dicom = get_dicom(self.dataset)
fields = get_fields(dicom)
# Test split action
actions = [{
"action": "SPLIT",
"field": "PatientID",
"value": 'by="^";minlength=4'
}]
expected_names = dicom.get("PatientID").split("^")
actual = extract_values_list(dicom, actions)
self.assertEqual(actual, expected_names)
# Test field action
actions = [{"action": "FIELD", "field": "startswith:Operator"}]
expected_operator = [
x.element.value for uid, x in fields.items()
if x.element.keyword.startswith("Operator")
]
actual = extract_values_list(dicom, actions)
self.assertEqual(actual, expected_operator)
print("Test deid.dicom.groups extract_fields_list")
actions = [{"action": "FIELD", "field": "contains:Instance"}]
expected = {
uid: x
for uid, x in fields.items() if "Instance" in x.element.keyword
}
actual = extract_fields_list(dicom, actions)
for uid in expected:
assert uid in actual
# Get identifiers for file
ids = get_identifiers(dicom)
self.assertTrue(isinstance(ids, dict))
# Add keys to be used for replace to ids - these first are for values
parser = DicomParser(dicom, recipe=self.deid)
parser.define("cookie_names", expected_names)
parser.define("operator_names", expected_operator)
# This is for fields
parser.define("instance_fields", expected)
parser.define("id", "new-cookie-id")
parser.define("source_id", "new-operator-id")
parser.parse()
# Were the changes made?
assert parser.dicom.get("PatientID") == "new-cookie-id"
assert parser.dicom.get("OperatorsName") == "new-operator-id"
# Instance fields should be removed based on recipe
for uid, field in parser.lookup["instance_fields"].items():
self.assertTrue(field.element.keyword not in parser.dicom)
# Start over
dicom = get_dicom(self.dataset)
# We need to provide ids with variables "id" and "source_id"
ids = {
dicom.filename: {
"id": "new-cookie-id",
"source_id": "new-operator-id"
}
}
# Returns list of updated dicom, since save is False
replaced = replace_identifiers(dicom,
save=False,
deid=self.deid,
ids=ids)
cleaned = replaced.pop()
self.assertEqual(cleaned.get("PatientID"), "new-cookie-id")
self.assertEqual(cleaned.get("OperatorsName"), "new-operator-id")