Example #1
0
def test_orthanc_ep(setup_orthanc0):

    logging.debug("Test Orthanc EP")

    O = Orthanc()
    print(O)
    O.check()
Example #2
0
def collect(ctx, project, data_path, source, domain, dest, anonymize,
            subpath_depth):
    """Create a PROJECT key at DATA_PATH, then pull data from
    SOURCE and send to DEST."""
    services = ctx.obj.get('services')

    click.echo(click.style('Collect DICOM data', underline=True, bold=True))

    C = Collector()

    _source = services[source]
    source_inst = Orthanc(**_source)

    if not dest:
        path = data_path / Path("data")
        dest_inst = DcmDir(path=path,
                           subpath_width=2,
                           subpath_depth=subpath_depth)
    elif dest.startswith("path:"):
        path = dest.split(":")[-1]
        dest_inst = DcmDir(path=path,
                           subpath_width=2,
                           subpath_depth=subpath_depth)
    else:
        _dest = services[dest]
        dest_inst = Orthanc(**_dest)

    C.run(project, data_path, source_inst, domain, dest_inst, anonymize)
Example #3
0
def test_site_submission(setup_orthanc0):
    reset_mock_seed()

    O = Orthanc()

    assert (O.check())

    n_instances_init = O.gateway.statistics()["CountInstances"]

    logging.debug(O.gateway.statistics())

    site_desc = yaml.load(sample_site_desc)

    H = MockSite.Factory.create(desc=site_desc)[0]

    try:
        with timeout(15):
            print("Starting mock site")
            H.run(pacs=O)
    except:
        print("Stopping mock site")

    n_instances = O.gateway.statistics()["CountInstances"]

    # At least 500 new instances arrived in the last 15 seconds
    assert (n_instances > n_instances_init + 500)
Example #4
0
    def put_accession(self, accession_number, dest: Orthanc):

        dixels = self.find_items_for(accession_number)
        for d in dixels:
            # logging.debug(type(d))
            d = self.filehandler.get(d, view="file")
            dest.put(d)
Example #5
0
def test_orthanc_upload(setup_orthanc0):

    logging.debug("Test Orthanc Upload")

    O = Orthanc()

    dicom_dir = find_resource("resources/dcm")
    D = DcmDir(path=dicom_dir)
    d = D.get("IM2263", view=DixelView.TAGS_FILE)

    O.put(d)

    q = {"PatientID": "AW15119516.678.1392297407"}
    result = O.find(q)

    if result:
        id = result[0]

    logging.debug(id)

    result = O.exists(id)

    logging.debug(result)

    assert (result)

    O.delete(d)

    result = O.exists(id)
    assert (not result)
Example #6
0
def fiup(ctx, collection, path, registry, dest, pool_size):
    """Collect files in a study by COLLECTION (accession number) using a
    PATH REGISTRY, and send to DEST."""
    services = ctx.obj.get('services')

    click.echo(click.style('Upload Registered Files by Accession Number',
                           underline=True, bold=True))

    file_indexer = FileIndexer(pool_size=pool_size)
    R = Redis(**services[registry])
    O = Orthanc(**services[dest])

    if collection != "ALL":

        result = file_indexer.upload_collection(
            collection=collection,
            basepath=path,
            registry=R,
            dest=O
        )

    else:

        items = file_indexer.items_on_path(basepath=path, registry=R)
        click.echo('Expecting {} items on path'.format(len(items)))

        result = file_indexer.upload_path(
            basepath=path,
            registry=R,
            dest=O
        )

    click.echo(result)
Example #7
0
 def poll(self, dest: Orthanc = None):
     if datetime.now() > self._next_study:
         print("{}: Study ready to push".format(self.station_name))
         study = self.gen_study()
         pprint(study.tags)
         for d in study.instances():
             d.gen_file()
             if dest:
                 dest.put(d)
         delay = 60 * 60 / self.studies_per_hour
         R = random.Random()
         delay = R.randint(int(0.5 * delay), int(1.5 * delay))
         self._next_study = datetime.now() + timedelta(seconds=delay)
         print("{}: Next study at {}".format(self.station_name,
                                             self._next_study))
     else:
         logging.debug("No study, next at {}".format(self._next_study))
         time.sleep(self.action_interval)
Example #8
0
def mock_runner():
    """Generates a single CR study and sends to the mock pacs"""

    reset_mock_seed()
    print("Starting mock site")
    O = Orthanc()
    desc = yaml.load(site_desc)
    H = MockSite.Factory.create(desc=desc)[0]
    H.run(pacs=O)
    print("Stopping mock site")
Example #9
0
def test_upload(setup_redis, setup_orthanc0):

    print("Testing upload speed")

    R = Redis()
    O = Orthanc()
    O.clear()

    dcmfdx = FileIndexer(pool_size=15)

    dcmfdx.upload_path(
        basepath=path,
        registry=R,
        dest=O
    )

    pprint(O.gateway.statistics())

    """
Example #10
0
    def make_key(self, ids, source: Orthanc, domain: str) -> set:

        print("Making key")

        # Minimal data for oid and sham plus study and series desc
        def mkq(accession_num):
            return {
                "PatientName": "",
                "PatientID": "",
                "PatientBirthDate": "",
                "PatientSex": "",
                "AccessionNumber": accession_num,
                "StudyDescription": "",
                "StudyInstanceUID": "",
                "StudyDate": "",
                "StudyTime": "",
            }

        items = set()
        for id in ids:

            q = mkq(id)

            try:
                r = source.rfind(q, domain, level=DicomLevel.STUDIES)
            except:
                r = None

            if not r:
                print("Failed to collect an id")
                continue

            tags = {
                "PatientName": r[0]["PatientName"],
                "PatientID": r[0]["PatientID"],
                "PatientBirthDate": r[0]["PatientBirthDate"],
                "PatientSex": r[0]["PatientSex"],
                "AccessionNumber": r[0]["AccessionNumber"],
                "StudyDescription": r[0]["StudyDescription"],
                "StudyInstanceUID": r[0]["StudyInstanceUID"],
                "StudyDate": r[0]["StudyDate"],
                "StudyTime": r[0]["StudyTime"]
            }

            d = Dixel(tags=tags)
            e = ShamDixel.from_dixel(d)

            items.add(e)
            print("Found {} items".format(len(items)))

            logging.debug(e)

        return items
Example #11
0
def orth_test_runner():
    O = Orthanc()
    dicom_dir = find_resource("resources/dcm")
    D = DcmDir(path=dicom_dir)

    print("Starting script")
    time.sleep(1)
    d = D.get("IM2263", view=DixelView.FILE)
    O.put(d)
    O.check()
    print("Ending script")
Example #12
0
def _handle_instance_in_dcm_dir(item: Dixel, orth: Orthanc, salt: str):

    orth.put(item)
    anon = ShamDixel.from_dixel(item, salt=salt)
    afile = orth.anonymize(anon, replacement_map=anon.orthanc_sham_map())
    anon.file = afile
    orth.put(anon)
    orth.delete(item)

    anon_study_id = anon.sham_parent_oid(DCMLv.STUDIES)
    logging.debug(anon_study_id)
    logging.debug(tagged_studies)
    if anon_study_id not in tagged_studies:
        logging.debug("Tagging parent study: {}".format(anon_study_id))
        siren_info = pack_siren_info(anon)
        orth.gateway.put_metadata(anon_study_id, DCMLv.STUDIES, "siren_info",
                                  siren_info)
        tagged_studies.append(anon_study_id)
Example #13
0
def mock(ctx, desc, dest):
    """Generate synthetic studies on a schedule according to a site
    description DESC.  Studies are optionally forwarded to an endpoint DEST."""
    services = ctx.obj.get('services')

    click.echo(
        click.style('Generate mock DICOM data', underline=True, bold=True))

    if not desc:
        desc = sample_site_desc
    desc = yaml.load(desc)

    H = MockSite.Factory.create(desc=desc)

    O = None
    if dest:
        _desc = services[dest]
        O = Orthanc(**_desc)

    for h in H:
        h.run(pacs=O)
Example #14
0
def test_mock_svcs():

    C.start_swarm()
    C.clean_swarm()

    c = C.api_client()

    admin_stack = find_resource("platform/docker-stacks/admin/admin-stack.yml")
    cmd = ["docker", "stack", "deploy", "-c", admin_stack, "admin"]
    subprocess.run(cmd)

    service_names = [x['Spec']['Name'] for x in c.services()]

    assert ("admin_portainer" in service_names)
    assert ("admin_traefik" in service_names)

    mock_stack = find_resource(
        "platform/docker-stacks/diana-workers/mock-stack.yml")
    cmd = ["docker", "stack", "deploy", "-c", mock_stack, "mock"]

    # Don't forget to set the password, or docker-compose will
    # interpret it as empty rather than default
    os.environ["ORTHANC_PASSWORD"] = "******"
    subprocess.run(cmd)

    service_names = [x['Spec']['Name'] for x in c.services()]

    assert ("mock_diana-mock" in service_names)
    assert ("mock_orthanc-mock" in service_names)

    # Pause to generate some data
    time.sleep(20)

    O = Orthanc(path="orthanc-mock", port=80)
    info = O.gateway._get("statistics")

    # At least 100 new images in the PACS
    assert (info['CountInstances'] > 100)

    C.clean_swarm()
Example #15
0
def test_anon(setup_orthanc0):
    O = Orthanc()
    dicom_dir = find_resource("resources/dcm")
    D = DcmDir(path=dicom_dir)
    d = D.get("IM2263", view=DixelView.TAGS_FILE)
    O.put(d)

    d.tags["AccessionNumber"] = "123456"
    d.tags["PatientBirthDate"] = "20000101"
    d.tags["PatientID"] = "ABC"
    d.tags["PatientName"] = "XYZ"
    d.level = DicomLevel.STUDIES
    e = ShamDixel.from_dixel(d)
    rep = e.orthanc_sham_map()

    O.anonymize("959e4e9f-e954be4e-11917c87-09d0f98f-7cc39128",
                level=DicomLevel.STUDIES,
                replacement_map=rep)
Example #16
0
    opts = parse_args()

    if opts.command == "index":
        with open(opts.secrets) as f:
            services = yaml.safe_load(f)
        redis_conf = services['redis']

        x = FileIndexer(location=opts.location, redis_conf=redis_conf)

        if opts.clear_cache:
            x.cache.clear()

        if opts.walk_type == "orthanc":
            x.run_orthanc(relpath=opts.relpath)
        else:
            x.run(relpath=opts.relpath, rex=opts.rex)

    elif opts.command == "restore":
        with open(opts.secrets) as f:
            services = yaml.safe_load(f)
        redis_conf = services[opts.redis_service]
        orthanc_conf = services[opts.destination_service]

        x = FileIndexer(location=opts.location, redis_conf=redis_conf)
        orthanc = Orthanc(**orthanc_conf)
        x.put_accession(accession_number=opts.accession_number, dest=orthanc)

    else:

        print("Valid commands are index or restore, see help for more info")
Example #17
0
    opts = parse_args()

    start = opts.start
    incr = opts.incr
    proxy_name = opts.proxy
    proxy_domain = opts.proxy_domain
    index_name = opts.index
    index_domain = opts.index_domain
    index_hec = opts.index_hec
    repeat_while = opts.repeat_while
    secrets_fn = opts.secrets

    with open(secrets_fn, 'r') as f:
        secrets = yaml.safe_load(f)

    proxy = Orthanc(**secrets[proxy_name])
    index = Splunk(**secrets[index_name])

    # TODO: Need to add a harvester with a delay that resets its time interval relative to now

    H = DoseReportHarvester(source=proxy,
                            source_domain=proxy_domain,
                            dest=index,
                            dest_domain=index_domain,
                            dest_hec=index_hec,
                            start=start,
                            incr=incr,
                            repeat_while=repeat_while)

    H.run()
Example #18
0
    def collect_items(self, source_query, index_query):

        w = self.surveyor.find_items(source_query, index_query)
        self.porter.move_items(w)


if __name__ == "__main__":

    logging.basicConfig(level=logging.DEBUG)

    # ------------------------------------------
    # Inventory a proxied DICOM node
    # ------------------------------------------

    proxy = Orthanc("http://locahost:8042")
    proxied_aet = "pacs"

    s = Surveyor(proxy, proxied_aet)
    w = s.find_items(source_query={"StudyDate": "01012018"})

    # ------------------------------------------
    # Lazy inventory (only new items)
    # ------------------------------------------

    splunk = Splunk("http://localhost:8000")
    splunk_kwargs = {"index": "dicom", "hec": None}

    t = Surveyor(proxy,
                 source_domain=proxied_aet,
                 dest=splunk,
Example #19
0
    def register_anon_and_save(self, dest):
        self.handlers.append(self.source.get, view="file")
        self.handlers.append(dest.put)
        self.handlers.append(dest.anonymize, remove=True)

    def register_index(self, discovered):
        self.handlers.append(self.source.get)
        self.handlers.append(discovered.put)


if __name__ == "__main__":

    logging.basicConfig(level=logging.DEBUG)

    source = DicomFile("/data/incoming/ftp")
    dest = Orthanc()
    observer = Observer()

    curator = Curator(source=source, observer=observer)
    curator.register_anon_and_save(dest)

    curator.watch()

exit()


@attr.s(hash=False)
class Watcher(object):
    uid = attr.ib(factory=uuid.uuid4)
    logger = attr.ib(init=False)
Example #20
0
    def pull_and_save(self,
                      items: Iterable,
                      source: Orthanc,
                      domain: str,
                      dest: DcmDir,
                      anonymize=False):
        def mkq(d: Dixel):
            return {"StudyInstanceUID": d.tags["StudyInstanceUID"]}

        for d in items:

            working_level = DicomLevel.STUDIES

            if anonymize:

                if working_level == DicomLevel.SERIES:
                    d_fn = "{}-{}.zip".format(
                        d.meta["ShamAccessionNumber"][0:6],
                        d.meta["ShamSeriesDescription"])
                else:
                    d_fn = "{}.zip".format(d.meta["ShamAccessionNumber"][0:16])

            else:

                if working_level == DicomLevel.SERIES:
                    d_fn = "{}-{}-{}.zip".format(
                        d.tags["PatientName"][0:6],
                        d.tags["AccessionNumber"][0:8],
                        d.tags["SeriesDescription"])
                else:
                    d_fn = "{}-{}.zip".format(d.tags["PatientName"][0:6],
                                              d.tags["AccessionNumber"][0:8])

            if dest.exists(d_fn):
                logging.debug("SKIPPING {}".format(d.tags["PatientName"]))
                continue

            if not source.exists(d):
                source.rfind(mkq(d),
                             domain,
                             level=working_level,
                             retrieve=True)
            else:
                logging.debug("SKIPPING PULL for {}".format(
                    d.tags["PatientName"]))

            if anonymize:
                try:
                    replacement_map = ShamDixel.orthanc_sham_map(d)

                    anon_id = source.anonymize(d,
                                               replacement_map=replacement_map)

                    e = source.get(anon_id,
                                   level=working_level,
                                   view=DixelView.FILE)
                    e.meta["FileName"] = d_fn
                    logging.debug(e)

                    dest.put(e)
                    source.delete(e)

                except (HTTPError, GatewayConnectionError) as e:
                    logging.error("Failed to anonymize dixel")
                    logging.error(e)
                    with open("errors.txt", "a+") as f:
                        f.write(d.tags["AccessionNumber"] + "\n")

            else:
                d = source.get(d, level=working_level, view=DixelView.FILE)
                dest.put(d)

            try:
                source.delete(d)
            except GatewayConnectionError as e:
                logging.error("Failed to delete dixel")
                logging.error(e)
Example #21
0
import os, yaml
from diana.apis import DcmDir, Orthanc

D = DcmDir(path="/data")

svcs_t = os.environ.get("DIANA_SERVICES")
svcs = yaml.load(svcs_t)

print(svcs)

args = svcs.get("renalstone")
O = Orthanc(**args)

for d in D.files(rex="*.zip"):

    file_set = D.get_zipped(d)

    for f in file_set:

        O.put(f)
Example #22
0
    def pull_and_send(self,
                      items: Iterable,
                      source: Orthanc,
                      domain: str,
                      dest: Orthanc,
                      anonymize=False):
        def mkq(d: Dixel):
            return {"StudyInstanceUID": d.tags["StudyInstanceUID"]}

        for d in items:

            sham_oid = ShamDixel.sham_oid(d)
            logging.debug(sham_oid)
            if dest.exists(sham_oid):
                logging.debug("SKIPPING {}".format(d.tags["PatientName"]))
                continue

            if not source.exists(d):
                source.rfind(mkq(d),
                             domain,
                             level=DicomLevel.STUDIES,
                             retrieve=True)
            else:
                logging.debug("SKIPPING PULL for {}".format(
                    d.tags["PatientName"]))

            replacement_map = ShamDixel.orthanc_sham_map(d)
            anon_id = source.anonymize(d, replacement_map=replacement_map)

            source.psend(anon_id, dest)
            source.delete(anon_id)
            source.delete(d)
Example #23
0
        logging.debug(accession_number)
        return self.cache.sget(accession_number)

    def put_accession(self, accession_number, dest: Orthanc):

        dixels = self.find_items_for(accession_number)
        for d in dixels:
            # logging.debug(type(d))
            d = self.filehandler.get(d, view="file")
            dest.put(d)


if __name__ == "__main__":

    logging.basicConfig(level=logging.DEBUG)

    with open("secrets/lifespan_services.yml") as f:
        services = yaml.safe_load(f)
    redis_conf = services['redis']

    x = FileIndexer(location="/Users/derek/data/DICOM", redis_conf=redis_conf)

    # x.cache.clear()
    # x.run(relpath="anon.chest_abd_pelvis", rex="IM*")  # Has no a/n
    # x.run(relpath="airway phantom", rex="IM*")
    # x.run_orthanc(relpath="Christianson")

    orthanc = Orthanc()
    x.put_accession("rphy10252012", orthanc)  # airway phant
    x.put_accession("4758606", orthanc)  # christianson
Example #24
0
def test_psend(setup_orthanc0, setup_orthanc1):

    O = Orthanc(peername="peer0")
    print(O)
    O.check()

    O2 = Orthanc(port=8043, peername="peer0")
    print(O2)
    O2.check()

    dicom_dir = find_resource("resources/dcm")
    D = DcmDir(path=dicom_dir)

    d = D.get("IM2263", view=DixelView.TAGS_FILE)
    O2.put(d)

    logging.debug(O2.gateway._get("peers"))

    O2.psend(d.oid(), O)

    e = O.get(d.oid(), level=DicomLevel.INSTANCES)

    logging.debug(e)

    assert d.oid() == e.oid()
Example #25
0
    parser.add_argument("-r", "--series", default=None)

    opts = parser.parse_args()

    return opts


if __name__ == "__main__":

    opts = parse_args()

    logging.basicConfig(level=logging.DEBUG)

    with open(opts.secrets, "r") as f:
        services = yaml.safe_load(f)

    orthanc = Orthanc(**services[opts.proxy])

    if not opts.series:
        d = Dixel(meta={"AccessionNumber": opts.accession},
                  level=DicomLevel.STUDIES)

    else:
        d = Dixel(meta={
            "AccessionNumber": opts.accession,
            "SeriesDescription": opts.series
        },
                  level=DicomLevel.SERIES)

    orthanc.find_item(d, domain=opts.domain, retrieve=True)