def anonymize_and_upload(R: DicomRegistry, D: DicomDirectory, O: Orthanc):
for d in R.instances.values():
ser_mhash = Dixel.from_child(d).mhash
# Find the real ser
ser = R.get(ser_mhash, dlvl=DLv.SERIES)
stu_mhash = Dixel.from_child(d, DLv.STUDY).mhash
# Find the real stu
stu = R.get(stu_mhash, dlvl=DLv.STUDY)
m = orthanc_sham_map(
stu.mhash,
stu.dhash,
# Try patient id first, fall back to patient name, or use "UNKNOWN"
patient_id=stu.tags.get("PatientID",
stu.tags.get("PatientName", "UNKNOWN")),
stu_dt=stu.timestamp,
ser_mhash=ser.mhash,
ser_dhash=ser.dhash,
ser_dt=ser.timestamp,
inst_mhash=d.mhash,
inst_dhash=d.dhash,
inst_dt=d.timestamp)
dd = D.get(d.meta["fp"], binary=True)
r = O.put(dd) # If no Patient ID, need to use returned value
oid = r['ID']
print(f"Putting {dd.main_tags()} in {oid}")
O.anonymize(oid,
replacement_map=m) # Anon inst no longer returns image?
O.delete(oid, dlvl=DLv.INSTANCE)
def get(self,
oid: UID,
dlvl: DLv = DLv.STUDY,
binary: bool = False,
view: str = None,
**kwargs) -> typ.Union[Dixel, typ.Dict]:
# Want to get the raw Orthanc metadata w child info
if view == "raw":
resource = f"{dlvl.opath()}/{oid}"
metadata = self.request(resource, **kwargs)
return metadata
if dlvl == DLv.INSTANCE:
resource = f"{dlvl.opath()}/{oid}/tags?simplify"
else:
resource = f"{dlvl.opath()}/{oid}/shared-tags?simplify"
tags = self.request(resource, **kwargs)
# pprint(tags)
d = Dixel.from_tags(tags, dlvl=dlvl)
if binary:
if dlvl > DLv.INSTANCE:
route = "archive"
else:
route = "file"
resource = f"{dlvl.opath()}/{oid}/{route}"
_file = self.request(resource, **kwargs)
if _file:
d.file = _file
return d
def put(self, inst: Dixel, **kwargs):
self.dhashes[self.t(inst, KTy.DHASH)] = inst
self.bhashes[self.t(inst, KTy.BHASH)] = inst
self.instances[self.t(inst)] = inst
ser = Dixel.from_child(inst, dlvl=DLv.SERIES)
if not self.series.get(self.t(ser)):
self.series[self.t(ser)] = ser
else:
self.series[self.t(ser)].add_child(inst)
stu = Dixel.from_child(inst, dlvl=DLv.STUDY)
if not self.studies.get(self.t(stu)):
self.studies[self.t(stu)] = stu
else:
self.studies[self.t(stu)].add_child(stu)
def measure_scout(dixel: Dixel):
"""Measure AP and Lateral dimensions from DICOM localizer images for SSDE calculations"""
# TODO: add a tag check for "LOCALIZER"
pixel_spacing = dixel.pixel_spacing
def measured_dimension(dixel):
"""
Determine measurement dimension
Could also try to use "degrees of azimuth" to identify lateral vs. AP:
- 90 its a lateral scout -> PA dimension
- 0,180 its an AP scout -> lateral dimension
"""
orientation = dixel.image_orientation_patient
if orientation[0] * orientation[0] > 0.2:
# This is a PA scout, which gives the lateral measurement
measured_dim = 'LATERAL'
elif orientation[1] * orientation[1] > 0.2:
# This is a lateral scout, which gives the AP measurement
measured_dim = 'AP'
else:
measured_dim = 'UNKNOWN'
return measured_dim
measured_dim = measured_dimension(dixel)
# Setup pixel array data
dcm_px = np.array(dixel.get_pixels(), dtype=np.float32)
# Determine weighted threshold separating tissue/non-tissue attenuations
# using a Gaussian Mixture Model
# thresh = np.mean(dcm_px[dcm_px>0])
gmm = GMM(2).fit(dcm_px[dcm_px > -1024].reshape(-1, 1))
thresh = np.sum(gmm.weights_[::-1] * gmm.means_.ravel())
# logging.debug(gmm.weights_[::-1])
# logging.debug(gmm.means_.ravel())
logging.debug("Threshold: {0}".format(thresh))
# Compute avg width based on unmasked pixels
mask = dcm_px > thresh
px_counts = np.sum(mask, axis=1)
avg_px_count = np.mean(px_counts[px_counts > 0])
# Across image spacing is 2nd component (axis 1)
d_avg = avg_px_count * pixel_spacing[1] / 10
logging.debug("Average {0} width: {1}".format(measured_dim, d_avg))
return (measured_dim, d_avg)
def make_key(self, ids, source: Orthanc, domain: str) -> set:
print("Making key")
# Minimal data for oid and sham plus study and series desc
def mkq(accession_num):
return {
"PatientName": "",
"PatientID": "",
"PatientBirthDate": "",
"PatientSex": "",
"AccessionNumber": accession_num,
"StudyDescription": "",
"StudyInstanceUID": "",
"StudyDate": "",
"StudyTime": "",
}
items = set()
for id in ids:
q = mkq(id)
try:
r = source.rfind(q, domain, level=DicomLevel.STUDIES)
except:
r = None
if not r:
print("Failed to collect an id")
continue
tags = {
"PatientName": r[0]["PatientName"],
"PatientID": r[0]["PatientID"],
"PatientBirthDate": r[0]["PatientBirthDate"],
"PatientSex": r[0]["PatientSex"],
"AccessionNumber": r[0]["AccessionNumber"],
"StudyDescription": r[0]["StudyDescription"],
"StudyInstanceUID": r[0]["StudyInstanceUID"],
"StudyDate": r[0]["StudyDate"],
"StudyTime": r[0]["StudyTime"]
}
d = Dixel(tags=tags)
e = ShamDixel.from_dixel(d)
items.add(e)
print("Found {} items".format(len(items)))
logging.debug(e)
return items
def get_pixels(dixel: Dixel, imsize=224):
pixels = dixel.get_pixels()
if dixel.meta.get("PhotometricInterpretation") == "MONOCHROME1":
pixels = np.invert(pixels.astype("uint16"))
pixels = pixels.astype("float32")
pixels -= np.min(pixels)
pixels /= np.max(pixels)
pixels *= 255.
pixels = resize_image(pixels, imsize, verbose=False)
return pixels
def test_redis_index(setup_redis):
R = Redis()
d = Dixel(meta={"FileName": "my_file"},
tags={"AccessionNumber": "100"})
R.add_to_collection(d, item_key="FileName")
logging.debug( R.collections() )
assert("100" in R.collections() )
logging.debug( R.collected_items("100") )
assert("my_file" in R.collected_items("100") )
def test_orthanc_upload():
O = Orthanc()
O.clear()
ROOTP = "~/data/dcm/ibis2"
D = DicomDirectory(ROOTP)
d = D.get("IM1", binary=True)
print(d.summary())
O.put(d)
g = O.get(d.oid(), dlvl=DLv.INSTANCE)
Dixel.comparator = CTy.METADATA
assert d == g
e = Dixel.from_child(d)
h = O.get(e.oid(), dlvl=DLv.SERIES)
assert e == h
f = Dixel.from_child(e)
i = O.get(f.oid(), dlvl=DLv.STUDY)
assert f == i
def test_csv(tmp_path):
fp = tmp_path / "tmp.csv"
ep0 = CsvFile(fp=fp)
ep0.fieldnames = ["_Age", "PatientName", "AccessionNumber"]
for i in range(10):
tags = {
"PatientName": "Subject {}".format(i),
"AccessionNumber": "Study {}".format(i)
}
meta = {
"Age": 20+i
}
ep0.dixels.add(Dixel(tags=tags, meta=meta))
ep0.write()
ep1 = CsvFile(fp=fp)
ep1.read()
d = ep1.dixels.pop()
logging.debug(d)
assert( d.meta['Age']>="20" )
assert( d.tags['PatientName'].startswith("Subject"))
logging.debug(ep1.fieldnames)
ep2 = CsvFile(fp=fp)
ep2.fieldnames = ['_ShamID', '_ShamBirthDate', '_ShamAccessionNumber', 'PatientName', 'AccessionNumber']
ShamDixel.REFERENCE_DATE = date(year=2018, month=1, day=1)
for d in ep0.dixels:
logging.debug(d)
ep2.dixels.add( ShamDixel.from_dixel(d) )
ep2.write()
ep2.fieldnames = "ALL"
ep2.write()
os.remove(fp)
def test_shams():
tags = {
"PatientName": "FOOABC^BAR^Z",
"PatientBirthDate": "20000101",
"StudyDate": "20180102",
"StudyTime": "120001",
"AccessionNumber": "12345678"
}
expected_meta = {
'ShamID': 'NTE3OTYH32XNES3LPNKR2U6CVOCZXDIL',
'ShamName': 'NADERMAN^TRACY^E',
'ShamBirthDate': "19991009",
'ShamStudyDateTime': datetime.datetime(2017, 10, 10, 12, 15, 3)
}
d = ShamDixel(tags=tags)
logging.debug(d)
assert (expected_meta.items() <= d.meta.items())
assert (d.meta["ShamName"] == "NADERMAN^TRACY^E")
assert (d.ShamStudyDate() == "20171010")
assert (
d.meta["ShamAccessionNumber"] == "25d55ad283aa400af464c76d713c07ad")
logging.debug(d.orthanc_sham_map())
expected_replacement_map = \
{'Replace': {'PatientName': 'NADERMAN^TRACY^E', 'PatientID': 'NTE3OTYH32XNES3LPNKR2U6CVOCZXDIL',
'PatientBirthDate': '19991009', 'AccessionNumber': '25d55ad283aa400af464c76d713c07ad',
'StudyInstanceUID': '1.2.826.0.1.3680043.10.43.62.716180617702.336145899646',
'StudyDate': '20171010', 'StudyTime': '121503'}, 'Keep': ['PatientSex', 'StudyDescription'],
'Force': True}
assert (d.orthanc_sham_map() == expected_replacement_map)
c = Dixel(tags=tags)
e = ShamDixel.from_dixel(c)
assert (d.meta["ShamName"] == e.meta["ShamName"])
f = ShamDixel.from_dixel(c, salt="FOO")
logging.debug(f.meta["ShamName"])
assert (d.meta["ShamName"] != f.meta["ShamName"])
def message(ctx, messenger, dixel, to_addrs, template, template_file, meta,
dryrun):
"""Call MESSENGER SEND with data from **DIXEL.meta and **META.
DIXEL may be a local path to an instance or a curl command to an Orthanc
\b
$ diana-cli message smtp path:/data/images/my_file.dcm \
-m "{{ tags.accession_number }}" -t "*****@*****.**"
$ diana-cli message smtp http:orthanc:orthanc@host/studies/oid... \
-m "{{ tags.accession_number }}" -t "*****@*****.**"
"""
services = ctx.obj.get('services')
click.echo(click.style('Calling messenger send', underline=True,
bold=True))
if dixel.startswith("path:"):
D = DcmDir()
d = D.get(dixel)
elif dixel.startswith("http"):
resp = requests.get(dixel)
if resp:
d = Dixel.from_orthanc(resp.json())
else:
click.echo(click.style("No such dixel {}".format(dixel), fg="red"))
exit(1)
if not services.get(messenger):
click.echo(
click.style("No such service {}".format(messenger), fg="red"))
exit(1)
_messenger = SmtpMessenger(**services[messenger])
_messenger.send(d, to_addrs=to_addrs, msg_t=template, dryrun=dryrun)
if dryrun:
click.echo(messenger.get(d, to_addrs=to_addrs, msg_t=template))
data_dir = "/Users/derek/Desktop/"
data_fn = "lscr_111218-050919.csv"
fpi = os.path.join( data_dir, data_fn )
fpo = os.path.join( data_dir, "lscr_out.csv")
worklist = CsvFile(fp=fpi)
worklist.read()
results = []
for d in worklist.dixels:
logging.debug(d)
study = Dixel.from_montage_csv(d.tags)
logging.debug(study)
# qdict = {"q": item.meta['Exam Unique ID'],
# "modality": 4,
# "exam_type": [8274, 8903], # 8119, 1997 (short term f/u)
# "start_date": "2018-01-01",
# "end_date": "2018-12-31"}
#
# results = M.find(qdict)
#
# if results:
#
# study = results.pop()
def put(self,
dx: Dixel,
link_duplicates=False,
ignore_errors=True,
**kwargs):
if dx.dlvl > DLv.INSTANCE:
if ignore_errors:
print("Can only register instances")
return
raise ValueError("Can only register instances")
if dx.dhash is None:
if ignore_errors:
print("Can only register hashed dixels")
return
raise AttributeError("Can only register hashed dixels")
_uid = self.hashes.get(dx.mhash)
if _uid is None:
_uid = uuid.uuid4().hex
self.meta[_uid] = dx.summary()
self.hashes[dx.mhash] = _uid
self.needs_cached = True # Flag a cache update
if self.hashes.get(dx.dhash) is None:
self.hashes[dx.dhash] = _uid
elif self.hashes.get(dx.dhash) != _uid:
if link_duplicates:
_uid1 = self.hashes.get(dx.dhash)
self.links[_uid1].add(_uid)
self.links[_uid].add(_uid1)
if ignore_errors:
print("Already seen this dhash under a different UID")
return
else:
raise ValueError(
"Already seen this dhash under a different UID")
if dx.bhash is not None:
if self.hashes.get(dx.bhash) is None:
self.hashes[dx.bhash] = _uid
elif self.hashes.get(dx.bhash) != _uid:
# This is always very bad!
raise ValueError(
"Already seen this bhash under a different UID")
def register_parent(par):
_uid = self.hashes.get(par.mhash)
if _uid is None:
_uid = uuid.uuid4().hex
self.meta[_uid] = par.summary()
self.hashes[par.mhash] = _uid
# self.hashes[par.uhash] = _uid
cur_dhash = self.meta[_uid]["dhash"] or ""
if cur_dhash in self.hashes:
del (self.hashes[cur_dhash])
new_dhash = hex_xor(dx.dhash, cur_dhash)
self.meta[_uid]["dhash"] = new_dhash
self.hashes[new_dhash] = _uid
if dx.bhash is not None:
cur_bhash = self.meta[_uid]["bhash"] or ""
if cur_bhash in self.hashes:
del (self.hashes[cur_bhash])
new_bhash = hex_xor(dx.bhash, cur_bhash)
self.meta[_uid]["bhash"] = new_bhash
self.hashes[new_bhash] = _uid
ser = Dixel.from_child(dx, dlvl=DLv.SERIES)
register_parent(ser)
stu = Dixel.from_child(dx, dlvl=DLv.STUDY)
register_parent(stu)
def parse_results(results, fn):
for i, entry in enumerate(results):
record_i = Dixel.from_montage_json(entry)
record_i.report.anonymize()
record_i.to_csv("{}/{}.csv".format(directory, fn[:-5]))