def test_load_recipe(self):
print("Case 1: Test loading default DeidRecipe")
recipe = DeidRecipe()
self.assertTrue(isinstance(recipe.deid, dict))
print("Checking basic sections are loaded")
print(recipe.deid.keys())
for section in ["header", "format", "filter"]:
self.assertTrue(section in recipe.deid)
print("Case 2: Loading from file")
recipe = DeidRecipe(self.deid)
def _prepare_replace_config(dicom_files, deid=None, config=None):
""" replace identifiers will replace dicom_files with data from ids based
on a combination of a config (default is remove all) and a user deid
Parameters
==========
dicom_files: the dicom file(s) to extract from
force: force reading the file (default True)
save: if True, save to file. Otherwise, return dicom objects
config: if None, uses default in provided module folder
overwrite: if False, save updated files to temporary directory
"""
if config is None:
config = "%s/config.json" % (here)
if not os.path.exists(config):
bot.error("Cannot find config %s, exiting" % (config))
if not isinstance(deid, DeidRecipe):
deid = DeidRecipe(deid)
config = read_json(config, ordered_dict=True)
if not isinstance(dicom_files, list):
dicom_files = [dicom_files]
return dicom_files, deid, config
def create_recipe(actions, fields=None, values=None):
"""helper method to create a recipe file"""
from deid.config import DeidRecipe
recipe = DeidRecipe()
# .clear() only supported Python 3.3 and after
del recipe.deid["header"][:]
recipe.deid["header"] = actions
if fields is not None:
recipe.deid["fields"] = fields
if values is not None:
recipe.deid["values"] = values
return recipe
def has_burned_pixels(dicom_files, force=True, deid=None):
"""has burned pixels is an entrypoint for has_burned_pixels_multi (for
multiple images) or has_burned_pixels_single (for one detailed repor)
We will use the MIRCTP criteria (see ref folder with the
original scripts used by CTP) to determine if an image is likely to have
PHI, based on fields in the header alone. This script does NOT perform
pixel cleaning, but returns a dictionary of results (for multi) or one
detailed result (for single)
"""
# if the user has provided a custom deid, load it
if not isinstance(deid, DeidRecipe):
if deid is None:
deid = "dicom"
deid = DeidRecipe(deid)
if isinstance(dicom_files, list):
return _has_burned_pixels_multi(dicom_files, force, deid)
return _has_burned_pixels_single(dicom_files, force, deid)
def __init__(self, dicom_file, recipe=None, config=None, force=True):
# Lookup for the dicom
self.lookup = {}
# Will be a list of DicomField
self.fields = {}
# Load default configuration, or a custom one
config = config or os.path.join(here, "config.json")
if not os.path.exists(config):
bot.error("Cannot find config %s, exiting" % (config))
self.config = read_json(config, ordered_dict=True)
# Deid can be a recipe or filename
if not isinstance(recipe, DeidRecipe):
recipe = DeidRecipe(recipe)
self.load(dicom_file, force=force)
self.recipe = recipe
def __init__(self,
output_folder=None,
add_padding=False,
margin=3,
deid=None,
font=None,
force=True):
if output_folder is None:
output_folder = get_temporary_name(prefix="clean")
if font is None:
font = self.default_font()
self.font = font
self.cmap = 'gray'
self.output_folder = output_folder
self.recipe = DeidRecipe(deid)
self.results = None
self.force = force
def __init__(self,
output_folder=None,
add_padding=False,
margin=3,
deid=None,
font=None,
force=True):
if output_folder is None:
output_folder = tempfile.mkdtemp()
if font is None:
font = self.default_font()
self.font = font
self.cmap = 'gray'
self.output_folder = output_folder
self.recipe = DeidRecipe(deid)
self.results = None
self.force = force
def main():
if len(sys.argv) is not 3:
print("argv")
sys.exit(1)
input_folder = sys.argv[1]
output_folder = sys.argv[2]
dicom_files = [
join(input_folder, dicom_file) for dicom_file in listdir(input_folder)
]
ids = get_identifiers(dicom_files)
# or use default conf, and then keep AccessionNumber
#recipe = DeidRecipe('deid.conf')
recipe = DeidRecipe()
#recipe.deid['header'].remove({'action': 'REMOVE', 'field': 'AccessionNumber'})
recipe.deid['header'].append({
'action': 'REMOVE',
'field': 'InstitutionName'
})
updated_ids = dict()
for image, fields in ids.items():
#fields['id'] = 'cookiemonster'
#fields['source_id'] = "cookiemonster-image-%s" %(count)
updated_ids[basename(image)] = fields
if not exists(output_folder):
try:
makedirs(output_folder)
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
cleaned_files = replace_identifiers(dicom_files=dicom_files,
deid=recipe,
ids=updated_ids,
output_folder=output_folder)
def test_get_functions(self):
recipe = DeidRecipe(self.deid)
# Format
self.assertEqual(recipe.get_format(), "dicom")
# Actions for header
print("Testing get_actions")
actions = recipe.get_actions()
self.assertTrue(isinstance(actions, list))
for key in ["action", "field", "value"]:
self.assertTrue(key in actions[0])
self.assertTrue(recipe.has_actions())
# Filters
print("Testing get_filters")
filters = recipe.get_filters()
self.assertTrue(isinstance(filters, dict))
# whitelist, blacklist, graylist
for key in recipe.ls_filters():
self.assertTrue(key in filters)
recipe = DeidRecipe()
filters = recipe.get_filters()
self.assertTrue(isinstance(filters["whitelist"], list))
# Test that each filter has a set of filters, coords, name
for key in ["filters", "coordinates", "name"]:
self.assertTrue(key in filters["whitelist"][0])
# Each filter is a list of actions, name is string, coords are list
self.assertTrue(isinstance(filters["whitelist"][0]["filters"], list))
self.assertTrue(isinstance(filters["whitelist"][0]["name"], str))
self.assertTrue(
isinstance(filters["whitelist"][0]["coordinates"], list))
# Check content of the first filter
for key in ["action", "field", "operator", "InnerOperators", "value"]:
self.assertTrue(key in filters["whitelist"][0]["filters"][0])
# Fields and Values
print("Testing get_fields_lists and get_values_lists")
self.assertEqual(recipe.get_fields_lists(), None)
self.assertEqual(recipe.get_values_lists(), None)
self.assertEqual(recipe.ls_fieldlists(), [])
self.assertEqual(recipe.ls_valuelists(), [])
self.assertTrue(not recipe.has_fields_lists())
self.assertTrue(not recipe.has_values_lists())
# Load in recipe with values and fields
deid = os.path.abspath("%s/../examples/deid/deid.dicom-groups" %
self.pwd)
recipe = DeidRecipe(deid)
assert "values" in recipe.deid
assert "fields" in recipe.deid
self.assertTrue(isinstance(recipe.deid["values"], dict))
self.assertTrue(isinstance(recipe.deid["fields"], dict))
self.assertTrue(recipe.get_fields_lists() is not None)
self.assertTrue(recipe.get_values_lists() is not None)
self.assertEqual(recipe.ls_fieldlists(), ["instance_fields"])
self.assertEqual(recipe.ls_valuelists(),
["cookie_names", "operator_names"])
self.assertTrue(recipe.has_fields_lists())
self.assertTrue(recipe.has_values_lists())
import os
import csv
from deid.dicom import get_identifiers, replace_identifiers
from deid.config import DeidRecipe
recipe = DeidRecipe(deid='./deidProBCR.dicom')
def createFolder(folder):
try:
if not os.path.exists(folder):
os.makedirs(folder)
except OSError:
print('Error: Creating folder. ' + folder)
#source and destin directories
src = "/Volumes/TOSHIBA_EXT/ProBCR"
dst = "/Volumes/TOSHIBA_EXT/ProBCR_Deid"
createFolder(dst)
csv_deid_coding = []
listOfDirectories = []
listOfFiles = []
listOfIDs4Files = []
pString = "P"
for root, directories, fileNames in os.walk(src):
if root == "/Volumes/TOSHIBA_EXT/ProBCR/S50710":
# The process of flagging images comes down to writing a set of filters to
# check if each image meets some criteria of interest. For example, I might
# create a filter called "xray" that is triggered when the Modality is CT or XR.
# We specify these fliters in a simple text file called a "deid recipe." When
# you work with the functions, you have the choice to instantiate the object
# in advance, or just provide a path to a recipe file. We will walk through
# examples for both below, starting with working with a DeidRecipe object.
# If you aren't interested in this use case or just want to use a provided
# deid recipe file, continue to the step to replace_identifiers
#
##################################
# Create a DeidRecipe
from deid.config import DeidRecipe
recipe = DeidRecipe()
# Since we didn't load a custom deid recipe text file, we get a default
# WARNING No specification, loading default base deid.dicom
# You can add custom deid files with .load().
# We can look at the criteria loaded:
recipe.deid
# You can also provide your own deid recipe file, and in doing so, you
# won't load a default
path = os.path.abspath("%s/../examples/deid/" % get_installdir())
recipe = DeidRecipe(deid=path)
# You can also choose to load the default base with your own recipe
recipe = DeidRecipe(deid=path, base=True)
from deid.config import DeidRecipe
# This is supported for deid.dicom version 0.1.34
# This dicom has nested InstanceCreationDate fields
dicom_files = ['MR.dcm']
# They are extracted, and flattened in items
# 'ReferencedPerformedProcedureStepSequence__InstanceCreationDate': '20091124',
items = get_identifiers(dicom_files)
# Load in the recipe, we want to REPLACE InstanceCreationDate with a function
recipe = DeidRecipe('deid.dicom')
# Here is our function
def generate_uid(item, value, field):
'''This function will generate a uuid! You can expect it to be passed
the dictionary of items extracted from the dicom (and your function)
and variables, the original value (func:generate_uid) and the field
name you are applying it to.
'''
import uuid
prefix = field.lower().replace(' ', " ")
return prefix + "-" + str(uuid.uuid4())
# Add the function to each item to be found
for item in items:
# FORMAT dicom
# %header
# REPLACE StudyInstanceUID func:generate_uid
# REPLACE SeriesInstanceUID func:generate_uid
# ADD FrameOfReferenceUID func:generate_uid
#
# In the above we are saying we want to replace the fields above with the
# output from the generate_uid function, which is expected in the item dict
##################################
# Create the DeidRecipe Instance from deid.dicom
from deid.config import DeidRecipe
recipe = DeidRecipe("deid.dicom")
# To see an entire (raw in a dictionary) recipe just look at
recipe.deid
# What is the format?
recipe.get_format()
# dicom
# What actions do we want to do on the header?
recipe.get_actions()
"""
[{'action': 'REPLACE',
'field': 'StudyInstanceUID',
'value': 'func:generate_uid'},
{'action': 'REPLACE',
# The process of flagging images comes down to writing a set of filters to
# check if each image meets some criteria of interest. For example, I might
# create a filter called "xray" that is triggered when the Modality is CT or XR.
# We specify these fliters in a simple text file called a "deid recipe." When
# you work with the functions, you have the choice to instantiate the object
# in advance, or just provide a path to a recipe file. We will walk through
# examples for both below, starting with working with a DeidRecipe object.
# If you aren't interested in this use case or just want to use a provided
# deid recipe file, continue to the step to replace_identifiers
#
##################################
# Create a DeidRecipe
from deid.config import DeidRecipe
recipe = DeidRecipe()
# Since we didn't load a custom deid recipe text file, we get a default
# WARNING No specification, loading default base deid.dicom
# You can add custom deid files with .load().
# We can look at the criteria loaded:
recipe.deid
# You can also provide your own deid recipe file, and in doing so, you
# won't load a default
path = os.path.abspath("%s/../examples/deid/" % get_installdir())
recipe = DeidRecipe(deid=path)
# You can also choose to load the default base with your own recipe
recipe = DeidRecipe(deid=path, base=True)
# FORMAT dicom
# %header
# REPLACE StudyInstanceUID func:generate_uid
# REPLACE SeriesInstanceUID func:generate_uid
# ADD FrameOfReferenceUID func:generate_uid
#
# In the above we are saying we want to replace the fields above with the
# output from the generate_uid function, which is expected in the item dict
##################################
# Create the DeidRecipe Instance from deid.dicom
from deid.config import DeidRecipe
recipe = DeidRecipe('deid.dicom')
# To see an entire (raw in a dictionary) recipe just look at
recipe.deid
# What is the format?
recipe.get_format()
# dicom
# What actions do we want to do on the header?
recipe.get_actions()
'''
[{'action': 'REPLACE',
'field': 'StudyInstanceUID',
'value': 'func:generate_uid'},
{'action': 'REPLACE',
def __init__(self, registers, recipe_file):
self.registers = registers
self.recipe = DeidRecipe(deid=recipe_file)
from deid.config import DeidRecipe
# This is supported for deid.dicom version 0.1.34
# This dicom has nested InstanceCreationDate fields
dicom_files = ["MR.dcm"]
# They are extracted, and flattened in items
# 'ReferencedPerformedProcedureStepSequence__InstanceCreationDate': '20091124',
items = get_identifiers(dicom_files)
# Load in the recipe, we want to REPLACE InstanceCreationDate with a function
recipe = DeidRecipe("deid.dicom")
# Here is our function
def generate_date(item, value, field, dicom):
"""This function will generate a dicom uid! You can expect it to be passed
the dictionary of items extracted from the dicom (and your function)
and variables, the original value (func:generate_uid) and the field
object you are applying it to.
"""
return "20200608"
# Add the function to each item to be found
for item in items: