def test_orthanc_ep(setup_orthanc0):
logging.debug("Test Orthanc EP")
O = Orthanc()
print(O)
O.check()
def collect(ctx, project, data_path, source, domain, dest, anonymize,
subpath_depth):
"""Create a PROJECT key at DATA_PATH, then pull data from
SOURCE and send to DEST."""
services = ctx.obj.get('services')
click.echo(click.style('Collect DICOM data', underline=True, bold=True))
C = Collector()
_source = services[source]
source_inst = Orthanc(**_source)
if not dest:
path = data_path / Path("data")
dest_inst = DcmDir(path=path,
subpath_width=2,
subpath_depth=subpath_depth)
elif dest.startswith("path:"):
path = dest.split(":")[-1]
dest_inst = DcmDir(path=path,
subpath_width=2,
subpath_depth=subpath_depth)
else:
_dest = services[dest]
dest_inst = Orthanc(**_dest)
C.run(project, data_path, source_inst, domain, dest_inst, anonymize)
def test_site_submission(setup_orthanc0):
reset_mock_seed()
O = Orthanc()
assert (O.check())
n_instances_init = O.gateway.statistics()["CountInstances"]
logging.debug(O.gateway.statistics())
site_desc = yaml.load(sample_site_desc)
H = MockSite.Factory.create(desc=site_desc)[0]
try:
with timeout(15):
print("Starting mock site")
H.run(pacs=O)
except:
print("Stopping mock site")
n_instances = O.gateway.statistics()["CountInstances"]
# At least 500 new instances arrived in the last 15 seconds
assert (n_instances > n_instances_init + 500)
def put_accession(self, accession_number, dest: Orthanc):
dixels = self.find_items_for(accession_number)
for d in dixels:
# logging.debug(type(d))
d = self.filehandler.get(d, view="file")
dest.put(d)
def test_orthanc_upload(setup_orthanc0):
logging.debug("Test Orthanc Upload")
O = Orthanc()
dicom_dir = find_resource("resources/dcm")
D = DcmDir(path=dicom_dir)
d = D.get("IM2263", view=DixelView.TAGS_FILE)
O.put(d)
q = {"PatientID": "AW15119516.678.1392297407"}
result = O.find(q)
if result:
id = result[0]
logging.debug(id)
result = O.exists(id)
logging.debug(result)
assert (result)
O.delete(d)
result = O.exists(id)
assert (not result)
def fiup(ctx, collection, path, registry, dest, pool_size):
"""Collect files in a study by COLLECTION (accession number) using a
PATH REGISTRY, and send to DEST."""
services = ctx.obj.get('services')
click.echo(click.style('Upload Registered Files by Accession Number',
underline=True, bold=True))
file_indexer = FileIndexer(pool_size=pool_size)
R = Redis(**services[registry])
O = Orthanc(**services[dest])
if collection != "ALL":
result = file_indexer.upload_collection(
collection=collection,
basepath=path,
registry=R,
dest=O
)
else:
items = file_indexer.items_on_path(basepath=path, registry=R)
click.echo('Expecting {} items on path'.format(len(items)))
result = file_indexer.upload_path(
basepath=path,
registry=R,
dest=O
)
click.echo(result)
def poll(self, dest: Orthanc = None):
if datetime.now() > self._next_study:
print("{}: Study ready to push".format(self.station_name))
study = self.gen_study()
pprint(study.tags)
for d in study.instances():
d.gen_file()
if dest:
dest.put(d)
delay = 60 * 60 / self.studies_per_hour
R = random.Random()
delay = R.randint(int(0.5 * delay), int(1.5 * delay))
self._next_study = datetime.now() + timedelta(seconds=delay)
print("{}: Next study at {}".format(self.station_name,
self._next_study))
else:
logging.debug("No study, next at {}".format(self._next_study))
time.sleep(self.action_interval)
def mock_runner():
"""Generates a single CR study and sends to the mock pacs"""
reset_mock_seed()
print("Starting mock site")
O = Orthanc()
desc = yaml.load(site_desc)
H = MockSite.Factory.create(desc=desc)[0]
H.run(pacs=O)
print("Stopping mock site")
def test_upload(setup_redis, setup_orthanc0):
print("Testing upload speed")
R = Redis()
O = Orthanc()
O.clear()
dcmfdx = FileIndexer(pool_size=15)
dcmfdx.upload_path(
basepath=path,
registry=R,
dest=O
)
pprint(O.gateway.statistics())
"""
def make_key(self, ids, source: Orthanc, domain: str) -> set:
print("Making key")
# Minimal data for oid and sham plus study and series desc
def mkq(accession_num):
return {
"PatientName": "",
"PatientID": "",
"PatientBirthDate": "",
"PatientSex": "",
"AccessionNumber": accession_num,
"StudyDescription": "",
"StudyInstanceUID": "",
"StudyDate": "",
"StudyTime": "",
}
items = set()
for id in ids:
q = mkq(id)
try:
r = source.rfind(q, domain, level=DicomLevel.STUDIES)
except:
r = None
if not r:
print("Failed to collect an id")
continue
tags = {
"PatientName": r[0]["PatientName"],
"PatientID": r[0]["PatientID"],
"PatientBirthDate": r[0]["PatientBirthDate"],
"PatientSex": r[0]["PatientSex"],
"AccessionNumber": r[0]["AccessionNumber"],
"StudyDescription": r[0]["StudyDescription"],
"StudyInstanceUID": r[0]["StudyInstanceUID"],
"StudyDate": r[0]["StudyDate"],
"StudyTime": r[0]["StudyTime"]
}
d = Dixel(tags=tags)
e = ShamDixel.from_dixel(d)
items.add(e)
print("Found {} items".format(len(items)))
logging.debug(e)
return items
def orth_test_runner():
O = Orthanc()
dicom_dir = find_resource("resources/dcm")
D = DcmDir(path=dicom_dir)
print("Starting script")
time.sleep(1)
d = D.get("IM2263", view=DixelView.FILE)
O.put(d)
O.check()
print("Ending script")
def _handle_instance_in_dcm_dir(item: Dixel, orth: Orthanc, salt: str):
orth.put(item)
anon = ShamDixel.from_dixel(item, salt=salt)
afile = orth.anonymize(anon, replacement_map=anon.orthanc_sham_map())
anon.file = afile
orth.put(anon)
orth.delete(item)
anon_study_id = anon.sham_parent_oid(DCMLv.STUDIES)
logging.debug(anon_study_id)
logging.debug(tagged_studies)
if anon_study_id not in tagged_studies:
logging.debug("Tagging parent study: {}".format(anon_study_id))
siren_info = pack_siren_info(anon)
orth.gateway.put_metadata(anon_study_id, DCMLv.STUDIES, "siren_info",
siren_info)
tagged_studies.append(anon_study_id)
def mock(ctx, desc, dest):
"""Generate synthetic studies on a schedule according to a site
description DESC. Studies are optionally forwarded to an endpoint DEST."""
services = ctx.obj.get('services')
click.echo(
click.style('Generate mock DICOM data', underline=True, bold=True))
if not desc:
desc = sample_site_desc
desc = yaml.load(desc)
H = MockSite.Factory.create(desc=desc)
O = None
if dest:
_desc = services[dest]
O = Orthanc(**_desc)
for h in H:
h.run(pacs=O)
def test_mock_svcs():
C.start_swarm()
C.clean_swarm()
c = C.api_client()
admin_stack = find_resource("platform/docker-stacks/admin/admin-stack.yml")
cmd = ["docker", "stack", "deploy", "-c", admin_stack, "admin"]
subprocess.run(cmd)
service_names = [x['Spec']['Name'] for x in c.services()]
assert ("admin_portainer" in service_names)
assert ("admin_traefik" in service_names)
mock_stack = find_resource(
"platform/docker-stacks/diana-workers/mock-stack.yml")
cmd = ["docker", "stack", "deploy", "-c", mock_stack, "mock"]
# Don't forget to set the password, or docker-compose will
# interpret it as empty rather than default
os.environ["ORTHANC_PASSWORD"] = "******"
subprocess.run(cmd)
service_names = [x['Spec']['Name'] for x in c.services()]
assert ("mock_diana-mock" in service_names)
assert ("mock_orthanc-mock" in service_names)
# Pause to generate some data
time.sleep(20)
O = Orthanc(path="orthanc-mock", port=80)
info = O.gateway._get("statistics")
# At least 100 new images in the PACS
assert (info['CountInstances'] > 100)
C.clean_swarm()
def test_anon(setup_orthanc0):
O = Orthanc()
dicom_dir = find_resource("resources/dcm")
D = DcmDir(path=dicom_dir)
d = D.get("IM2263", view=DixelView.TAGS_FILE)
O.put(d)
d.tags["AccessionNumber"] = "123456"
d.tags["PatientBirthDate"] = "20000101"
d.tags["PatientID"] = "ABC"
d.tags["PatientName"] = "XYZ"
d.level = DicomLevel.STUDIES
e = ShamDixel.from_dixel(d)
rep = e.orthanc_sham_map()
O.anonymize("959e4e9f-e954be4e-11917c87-09d0f98f-7cc39128",
level=DicomLevel.STUDIES,
replacement_map=rep)
opts = parse_args()
if opts.command == "index":
with open(opts.secrets) as f:
services = yaml.safe_load(f)
redis_conf = services['redis']
x = FileIndexer(location=opts.location, redis_conf=redis_conf)
if opts.clear_cache:
x.cache.clear()
if opts.walk_type == "orthanc":
x.run_orthanc(relpath=opts.relpath)
else:
x.run(relpath=opts.relpath, rex=opts.rex)
elif opts.command == "restore":
with open(opts.secrets) as f:
services = yaml.safe_load(f)
redis_conf = services[opts.redis_service]
orthanc_conf = services[opts.destination_service]
x = FileIndexer(location=opts.location, redis_conf=redis_conf)
orthanc = Orthanc(**orthanc_conf)
x.put_accession(accession_number=opts.accession_number, dest=orthanc)
else:
print("Valid commands are index or restore, see help for more info")
opts = parse_args()
start = opts.start
incr = opts.incr
proxy_name = opts.proxy
proxy_domain = opts.proxy_domain
index_name = opts.index
index_domain = opts.index_domain
index_hec = opts.index_hec
repeat_while = opts.repeat_while
secrets_fn = opts.secrets
with open(secrets_fn, 'r') as f:
secrets = yaml.safe_load(f)
proxy = Orthanc(**secrets[proxy_name])
index = Splunk(**secrets[index_name])
# TODO: Need to add a harvester with a delay that resets its time interval relative to now
H = DoseReportHarvester(source=proxy,
source_domain=proxy_domain,
dest=index,
dest_domain=index_domain,
dest_hec=index_hec,
start=start,
incr=incr,
repeat_while=repeat_while)
H.run()
def collect_items(self, source_query, index_query):
w = self.surveyor.find_items(source_query, index_query)
self.porter.move_items(w)
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
# ------------------------------------------
# Inventory a proxied DICOM node
# ------------------------------------------
proxy = Orthanc("http://locahost:8042")
proxied_aet = "pacs"
s = Surveyor(proxy, proxied_aet)
w = s.find_items(source_query={"StudyDate": "01012018"})
# ------------------------------------------
# Lazy inventory (only new items)
# ------------------------------------------
splunk = Splunk("http://localhost:8000")
splunk_kwargs = {"index": "dicom", "hec": None}
t = Surveyor(proxy,
source_domain=proxied_aet,
dest=splunk,
def register_anon_and_save(self, dest):
self.handlers.append(self.source.get, view="file")
self.handlers.append(dest.put)
self.handlers.append(dest.anonymize, remove=True)
def register_index(self, discovered):
self.handlers.append(self.source.get)
self.handlers.append(discovered.put)
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
source = DicomFile("/data/incoming/ftp")
dest = Orthanc()
observer = Observer()
curator = Curator(source=source, observer=observer)
curator.register_anon_and_save(dest)
curator.watch()
exit()
@attr.s(hash=False)
class Watcher(object):
uid = attr.ib(factory=uuid.uuid4)
logger = attr.ib(init=False)
def pull_and_save(self,
items: Iterable,
source: Orthanc,
domain: str,
dest: DcmDir,
anonymize=False):
def mkq(d: Dixel):
return {"StudyInstanceUID": d.tags["StudyInstanceUID"]}
for d in items:
working_level = DicomLevel.STUDIES
if anonymize:
if working_level == DicomLevel.SERIES:
d_fn = "{}-{}.zip".format(
d.meta["ShamAccessionNumber"][0:6],
d.meta["ShamSeriesDescription"])
else:
d_fn = "{}.zip".format(d.meta["ShamAccessionNumber"][0:16])
else:
if working_level == DicomLevel.SERIES:
d_fn = "{}-{}-{}.zip".format(
d.tags["PatientName"][0:6],
d.tags["AccessionNumber"][0:8],
d.tags["SeriesDescription"])
else:
d_fn = "{}-{}.zip".format(d.tags["PatientName"][0:6],
d.tags["AccessionNumber"][0:8])
if dest.exists(d_fn):
logging.debug("SKIPPING {}".format(d.tags["PatientName"]))
continue
if not source.exists(d):
source.rfind(mkq(d),
domain,
level=working_level,
retrieve=True)
else:
logging.debug("SKIPPING PULL for {}".format(
d.tags["PatientName"]))
if anonymize:
try:
replacement_map = ShamDixel.orthanc_sham_map(d)
anon_id = source.anonymize(d,
replacement_map=replacement_map)
e = source.get(anon_id,
level=working_level,
view=DixelView.FILE)
e.meta["FileName"] = d_fn
logging.debug(e)
dest.put(e)
source.delete(e)
except (HTTPError, GatewayConnectionError) as e:
logging.error("Failed to anonymize dixel")
logging.error(e)
with open("errors.txt", "a+") as f:
f.write(d.tags["AccessionNumber"] + "\n")
else:
d = source.get(d, level=working_level, view=DixelView.FILE)
dest.put(d)
try:
source.delete(d)
except GatewayConnectionError as e:
logging.error("Failed to delete dixel")
logging.error(e)
import os, yaml
from diana.apis import DcmDir, Orthanc
D = DcmDir(path="/data")
svcs_t = os.environ.get("DIANA_SERVICES")
svcs = yaml.load(svcs_t)
print(svcs)
args = svcs.get("renalstone")
O = Orthanc(**args)
for d in D.files(rex="*.zip"):
file_set = D.get_zipped(d)
for f in file_set:
O.put(f)
def pull_and_send(self,
items: Iterable,
source: Orthanc,
domain: str,
dest: Orthanc,
anonymize=False):
def mkq(d: Dixel):
return {"StudyInstanceUID": d.tags["StudyInstanceUID"]}
for d in items:
sham_oid = ShamDixel.sham_oid(d)
logging.debug(sham_oid)
if dest.exists(sham_oid):
logging.debug("SKIPPING {}".format(d.tags["PatientName"]))
continue
if not source.exists(d):
source.rfind(mkq(d),
domain,
level=DicomLevel.STUDIES,
retrieve=True)
else:
logging.debug("SKIPPING PULL for {}".format(
d.tags["PatientName"]))
replacement_map = ShamDixel.orthanc_sham_map(d)
anon_id = source.anonymize(d, replacement_map=replacement_map)
source.psend(anon_id, dest)
source.delete(anon_id)
source.delete(d)
logging.debug(accession_number)
return self.cache.sget(accession_number)
def put_accession(self, accession_number, dest: Orthanc):
dixels = self.find_items_for(accession_number)
for d in dixels:
# logging.debug(type(d))
d = self.filehandler.get(d, view="file")
dest.put(d)
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
with open("secrets/lifespan_services.yml") as f:
services = yaml.safe_load(f)
redis_conf = services['redis']
x = FileIndexer(location="/Users/derek/data/DICOM", redis_conf=redis_conf)
# x.cache.clear()
# x.run(relpath="anon.chest_abd_pelvis", rex="IM*") # Has no a/n
# x.run(relpath="airway phantom", rex="IM*")
# x.run_orthanc(relpath="Christianson")
orthanc = Orthanc()
x.put_accession("rphy10252012", orthanc) # airway phant
x.put_accession("4758606", orthanc) # christianson
def test_psend(setup_orthanc0, setup_orthanc1):
O = Orthanc(peername="peer0")
print(O)
O.check()
O2 = Orthanc(port=8043, peername="peer0")
print(O2)
O2.check()
dicom_dir = find_resource("resources/dcm")
D = DcmDir(path=dicom_dir)
d = D.get("IM2263", view=DixelView.TAGS_FILE)
O2.put(d)
logging.debug(O2.gateway._get("peers"))
O2.psend(d.oid(), O)
e = O.get(d.oid(), level=DicomLevel.INSTANCES)
logging.debug(e)
assert d.oid() == e.oid()
parser.add_argument("-r", "--series", default=None)
opts = parser.parse_args()
return opts
if __name__ == "__main__":
opts = parse_args()
logging.basicConfig(level=logging.DEBUG)
with open(opts.secrets, "r") as f:
services = yaml.safe_load(f)
orthanc = Orthanc(**services[opts.proxy])
if not opts.series:
d = Dixel(meta={"AccessionNumber": opts.accession},
level=DicomLevel.STUDIES)
else:
d = Dixel(meta={
"AccessionNumber": opts.accession,
"SeriesDescription": opts.series
},
level=DicomLevel.SERIES)
orthanc.find_item(d, domain=opts.domain, retrieve=True)
