def test_get_dataset(self):
'''test_get_dataset will make sure we can load provided datasets
'''
print("Case 1: Ask for existing dataset.")
from deid.data import get_dataset
dataset = get_dataset('dicom-cookies')
self.assertTrue(os.path.exists(dataset))
print("Case 2: Ask for non existing dataset")
dataset = get_dataset('other-cookies')
self.assertEqual(dataset, None)
def setUp(self):
self.pwd = get_installdir()
self.deid = os.path.abspath("%s/../examples/deid/deid.dicom" %
self.pwd)
self.dataset = get_dataset("dicom-cookies")
self.tmpdir = tempfile.mkdtemp()
print("\n######################START######################")
def test_field_expansion(self):
print("Test deid.dicom.fields expand_field_expression")
from deid.dicom.fields import expand_field_expression
dicom = get_dicom(self.dataset)
contenders = dicom.dir()
print("Testing that field expansion works for basic tags")
expand_field_expression(dicom=dicom,
field="endswith:Time",
contenders=contenders)
print("Testing that field expansion works including private tags")
contenders += [e.tag for e in get_private(dicom)]
expand_field_expression(dicom=dicom,
field="endswith:Time",
contenders=contenders)
print("Testing that we can also search private tags based on numbers.")
fields = expand_field_expression(dicom=dicom,
field="contains:0019",
contenders=contenders)
# We should have a tag object in the list now!
assert isinstance(fields[0], BaseTag)
print("Testing nested private tags")
dataset = get_dataset("animals") # includes nested private tags
dicom = get_dicom(dataset)
def setUp(self):
self.pwd = get_installdir()
self.deid = os.path.abspath("%s/../examples/deid/deid.dicom" %
self.pwd)
self.dataset = get_dataset("animals") # includes private tags
self.tmpdir = tempfile.mkdtemp()
print("\n######################START######################")
def test_field_expansion(self):
print("Test deid.dicom.fields expand_field_expression")
from deid.dicom.fields import expand_field_expression
dicom = get_dicom(self.dataset)
contenders = get_fields(dicom)
print("Testing that field expansion works for basic tags")
fields = expand_field_expression(dicom=dicom,
field="endswith:Time",
contenders=contenders)
# The fields returned should end in time
for uid, field in fields.items():
assert field.name.endswith("Time")
print("Testing that we can also search private tags based on numbers.")
fields = expand_field_expression(dicom=dicom,
field="contains:0019",
contenders=contenders)
# The fields returned should include tag group or element 0019
for uid, field in fields.items():
assert "0019" in uid
print("Testing nested private tags")
dataset = get_dataset("animals") # includes nested private tags
dicom = get_dicom(dataset)
def main(args, parser):
# Global output folder
output_folder = args.outfolder
if output_folder is None:
output_folder = tempfile.mkdtemp()
# If a deid is given, check against format
if args.deid is not None:
params = load_deid(args.deid)
if params["format"] != args.format:
bot.error(
"Format in deid (%s) doesn't match choice here (%s) exiting." %
(params["format"], args.format))
# Get list of dicom files
base = args.input
if base is None:
bot.info("No input folder specified, will use demo dicom-cookies.")
base = get_dataset("dicom-cookies")
basename = os.path.basename(base)
dicom_files = list(
get_files(base)) # todo : consider using generator functionality
do_get = False
do_put = False
ids = None
if args.action == "all":
bot.info("GET and PUT identifiers from %s" % (basename))
do_get = True
do_put = True
elif args.action == "get":
do_get = True
bot.info("GET and PUT identifiers from %s" % (basename))
elif args.action == "put":
bot.info("PUT identifiers from %s" % (basename))
do_put = True
if args.ids is None:
bot.exit(
"To PUT without GET you must provide a json file with ids.")
ids = args.ids
# GET identifiers
if do_get is True:
ids = get_identifiers(dicom_files)
if do_put is True:
cleaned_files = replace_identifiers(
dicom_files=dicom_files,
ids=ids,
deid=args.deid,
overwrite=args.overwrite,
output_folder=output_folder,
)
bot.info("%s %s files at %s" %
(len(cleaned_files), args.format, output_folder))
def main(args, parser):
"""inspect currently serves to inspect the header fields of a set
of dicom files against a standard, and flag images that don't
pass the different levels of criteria
"""
# If a deid is given, check against format
deid = args.deid
if deid is not None:
params = load_deid(deid)
if params["format"] != args.format:
bot.error(
"Format in deid (%s) doesn't match choice here (%s) exiting."
% (params["format"], args.format)
)
# Get list of dicom files
base = args.folder
if base is None:
bot.info("No input folder specified, will use demo dicom-cookies.")
base = get_dataset("dicom-cookies")
dicom_files = list(
get_files(base, pattern=args.pattern)
) # todo : consider using generator functionality
result = has_burned_pixels(dicom_files, deid=deid)
print("\nSUMMARY ================================\n")
if result["clean"]:
bot.custom(
prefix="CLEAN", message="%s files" % len(result["clean"]), color="CYAN"
)
if result["flagged"]:
for group, files in result["flagged"].items():
bot.flag("%s %s files" % (group, len(files)))
if args.save:
folders = "-".join([os.path.basename(folder) for folder in base])
outfile = "pixel-flag-results-%s-%s.tsv" % (
folders,
datetime.datetime.now().strftime("%y-%m-%d"),
)
with open(outfile, "w") as filey:
filey.writelines("dicom_file\tpixels_flagged\tflag_list\treason\n")
for clean in result["clean"]:
filey.writelines("%s\tCLEAN\t\t\n" % clean)
for flagged, details in result["flagged"].items():
if details["flagged"] is True:
for result in details["results"]:
group = result["group"]
reason = result["reason"]
filey.writelines(
"%s\tFLAGGED\t%s\t%s\n" % (flagged, group, reason)
)
print("Result written to %s" % outfile)
def main(args, parser):
'''inspect currently serves to inspect the header fields of a set
of dicom files against a standard, and flag images that don't
pass the different levels of criteria
'''
# Global output folder
#output_folder = args.outfolder
#if output_folder is None:
# output_folder = tempfile.mkdtemp()
# If a deid is given, check against format
deid = args.deid
if deid is not None:
params = load_deid(deid)
if params['format'] != args.format:
bot.error(
"Format in deid (%s) doesn't match choice here (%s) exiting." %
(params['format'], args.format))
# Get list of dicom files
base = args.folder
if base is None:
bot.info("No input folder specified, will use demo dicom-cookies.")
base = get_dataset('dicom-cookies')
dicom_files = list(get_files(
base,
pattern=args.pattern)) # todo : consider using generator functionality
result = has_burned_pixels(dicom_files, deid=deid)
print('\nSUMMARY ================================\n')
if len(result['clean']) > 0:
bot.custom(prefix='CLEAN',
message="%s files" % len(result['clean']),
color="CYAN")
if len(result['flagged']) > 0:
for group, files in result['flagged'].items():
bot.flag("%s %s files" % (group, len(files)))
if args.save is True:
folders = '-'.join([os.path.basename(folder) for folder in base])
outfile = "pixel-flag-results-%s-%s.tsv" % (
folders, datetime.datetime.now().strftime('%y-%m-%d'))
with open(outfile, 'w') as filey:
filey.writelines('dicom_file\tpixels_flagged\tflag_list\treason\n')
for clean in result['clean']:
filey.writelines('%s\tCLEAN\t\t\n' % clean)
for flagged, details in result['flagged'].items():
if details['flagged'] is True:
for result in details['results']:
group = result['group']
reason = result['reason']
filey.writelines('%s\tFLAGGED\t%s\t%s\n' %
(flagged, group, reason))
print('Result written to %s' % outfile)
#!/usr/bin/env python3
# This is a complete example of inspecting pixels for PHI
# based on a deid.dicom specification
# https://pydicom.github.io/deid
# This will get a set of example cookie dicoms
from deid.dicom import get_files, has_burned_pixels
from pydicom import read_file
from deid.data import get_dataset
from deid.logger import bot
import os
bot.level = 3
base = get_dataset("dicom-cookies")
dicom_files = list(
get_files(base)) # todo : consider using generator functionality
results = has_burned_pixels(dicom_files=dicom_files, deid="examples/deid")
# The dictionary has a "clean" list, and a "flagged" list,
# Eg:
# {'clean': [],
# 'flagged': {'/home/vanessa/Documents/Dropbox/Code/dicom/deid/deid/data/dicom-cookies/image1.dcm': {'flagged': True,
# 'results': [{'coordinates': [],
# 'group': 'blacklist',
# 'reason': ' ImageType missing or ImageType empty '}]},
#!/usr/bin/env python3
from deid.dicom import get_files, replace_identifiers
from deid.utils import get_installdir
from deid.data import get_dataset
import os
# This is a complete example of doing de-identifiction. For details, see our docs
# https://pydicom.github.io/deid
# This will get a set of example cookie dicoms
base = get_dataset('dicom-cookies')
dicom_files = list(
get_files(base)) # todo : consider using generator functionality
# This is the function to get identifiers
from deid.dicom import get_identifiers
ids = get_identifiers(dicom_files)
#**
# Here you might save them in your special (IRB approvied) places
# And then provide replacement anonymous ids to put back in the data
# A cookie tumor example is below
#**
################################################################################
# The Deid Recipe
#
# The process of flagging images comes down to writing a set of filters to
# check if each image meets some criteria of interest. For example, I might
# RADIOLOGY ---------------------------------------------------
# This is an example script to upload data (images, text, metadata) to
# google cloud storage and datastore. Data MUST be de-identified
som.api.google.datastore import DataStoreClient as Client
import os
# Start google storage client for pmc-stanford
client = Client(bucket_name='radiology')
collection = client.create_collection(uid='IRB41449')
# Let's load some dummy data from deid
from deid.data import get_dataset
from deid.dicom import get_files
dicom_files = get_files(get_dataset('dicom-cookies'))
# Now de-identify to get clean files
from deid.dicom import get_identifiers, replace_identifiers
ids=get_identifiers(dicom_files)
updated_files = replace_identifiers(dicom_files=dicom_files,
ids=ids)
# Define some metadata for the entity
metadata = { "source_id" : "cookieTumorDatabase",
"id":"cookie-47",
"Modality": "cookie"}
# Upload the dataset
client.upload_dataset(images=updated_files,
collection=collection,
def setUp(self):
self.pwd = get_installdir()
self.deidpath = os.path.abspath("%s/tests/resources/" % self.pwd)
self.dataset = get_dataset("animals")
self.tmpdir = tempfile.mkdtemp()
print("\n######################START######################")