def list_datasets(host: str) -> List[dict]:
"""Get all datasets
Args:
host: Data node host IP
Yields:
list of datasets.
Examples:
>>> datasets = list_datasets(host="0.0.0.0/api/v1")
>>> list(datasets)[0]
>>> {
>>> "name": "datasets/testing"
>>> }
"""
configuration = datanode.Configuration(host=host)
offset = 0
limit = 10
with datanode.ApiClient(configuration) as api_client:
api_instance = dataset_api.DatasetApi(api_client)
# Obtain all clinical notes
next_page = True
while next_page:
datasets = api_instance.list_datasets(offset=offset, limit=limit)
# change from snake case to camel case
sanitized = api_client.sanitize_for_serialization(
datasets.datasets)
for dataset in sanitized:
yield dataset
next_page = datasets.links.next
offset += limit
def list_annotations(host: str, dataset_id: str,
annotation_store_id: str) -> Iterator[dict]:
"""List annotations
Args:
host: Data node host IP
dataset_id: Dataset Id
annotation_store_id: Annotation store Id
Yields:
Data node annotation objects
Examples:
>>> annotations = list_annotations(host="0.0.0.0/api/v1",
>>> dataset_id="awesome-dataset",
>>> annotation_store_id="awesome-annotation-store")
"""
configuration = datanode.Configuration(host=host)
offset = 0
limit = 10
with datanode.ApiClient(configuration) as api_client:
annotation_instance = annotation_api.AnnotationApi(api_client)
next_page = True
while next_page:
annotations = annotation_instance.list_annotations(
dataset_id, annotation_store_id, offset=offset, limit=limit)
# change from snake case to camel case
sanitized_annotations = api_client.sanitize_for_serialization(
annotations.annotations)
for annotation in sanitized_annotations:
yield annotation
next_page = annotations.links.next
offset += limit
def get_annotation(host: str, dataset_id: str, annotation_store_id: str,
annotation_id: str) -> Annotation:
"""Gets an annotation
Args:
host: Data node host IP
dataset_id: Dataset Id
annotation_store_id: Annotation store Id
annotation_id: Annotation Id
Returns:
Data node Annotation object
Examples:
>>> annotation = get_annotation(
>>> host="0.0.0.0/api/v1", dataset_id="awesome-dataset",
>>> annotation_store_id="awesome-annotation-store",
>>> annotation_id="awesome-annotation"
>>> )
"""
configuration = datanode.Configuration(host=host)
with datanode.ApiClient(configuration) as api_client:
annot_instance = annotation_api.AnnotationApi(api_client)
# get the annotation store
annotation_store_obj = annot_instance.get_annotation(
dataset_id, annotation_store_id, annotation_id)
return annotation_store_obj
def delete_dataset(host: str, dataset_id: str):
"""Deletes a dataset"""
configuration = datanode.Configuration(host=host)
# Enter a context with an instance of the API client
with datanode.ApiClient(configuration) as api_client:
# Create an instance of the API class
api_instance = dataset_api.DatasetApi(api_client)
api_instance.delete_dataset(dataset_id)
def _store_annotation(host: str, dataset_id: str, annotation_store_id: str,
annotation_id: str, annotation: dict) -> Annotation:
"""Store annotation
Args:
host: Data node host IP
dataset_id: Dataset Id
annotation_store_id: Annotation store Id
annotation: Annotation dict
Returns:
Data node Annotation object
Examples:
>>> example_annotation = {
>>> "annotationSource": {
>>> "resourceSource": {
>>> "name": "name"
>>> }
>>> },
>>> "textDateAnnotations": [
>>> {
>>> "dateFormat": "MM/DD/YYYY",
>>> "length": 10,
>>> "start": 42,
>>> "text": "10/26/2020"
>>> },
>>> {
>>> "dateFormat": "MM/DD/YYYY",
>>> "length": 10,
>>> "start": 42,
>>> "text": "10/26/2020"
>>> }
>>> ],
>>> "textPersonNameAnnotations": [],
>>> "textPhysicalAddressAnnotations": []
>>> }
>>> annotation = store_annotation(host="0.0.0.0/api/v1",
>>> dataset_id="awesome-dataset",
>>> annotation_store_id="awesome-annotation-store",
>>> annotation_id="awesome-id",
>>> annotation=example_annotation)
"""
configuration = datanode.Configuration(host=host)
with datanode.ApiClient(configuration) as api_client:
annotation_instance = annotation_api.AnnotationApi(api_client)
new_annotation = utils.change_keys(annotation,
utils.camelcase_to_snakecase)
annotation_obj = annotation_instance.create_annotation(
dataset_id=dataset_id,
annotation_store_id=annotation_store_id,
annotation_id=annotation_id,
annotation_create_request=new_annotation,
async_req=True)
return annotation_obj.get()
def store_dataset(host: str, dataset_id: str) -> Dataset:
"""Creates a dataset"""
configuration = datanode.Configuration(host=host)
# Enter a context with an instance of the API client
with datanode.ApiClient(configuration) as api_client:
# Create an instance of the API class
api_instance = dataset_api.DatasetApi(api_client)
body = {}
dataset = api_instance.create_dataset(dataset_id, body=body)
return dataset
def setup_method(self):
self.configuration = datanode.Configuration()
self.api = datanode.ApiClient()
self.mock_api = Mock()
self.host = "0.0.0.0"
self.config = patch.object(datanode,
"Configuration",
return_value=self.configuration)
self.api_client = patch.object(datanode, "ApiClient")
self.dataset_id = "awesome-dataset"
self.annotation_store_id = "annotation-store"
self.annotation_id = "awesome-annotation"
self.fhir_store_id = "fhir-store"
def list_notes(host: str, dataset_id: str, fhir_store_id: str) -> List[dict]:
"""Get all clinical notes for a dataset
Args:
host: Data node host IP
dataset_id: Dataset Id
fhir_store_id: FHIR store Id
Yields:
list of clinical notes.
Examples:
>>> notes = get_notes(host="0.0.0.0/api/v1",
>>> dataset_id="awesome-dataset",
>>> fhir_store_id="awesome-fhir-store")
>>> list(notes)[0]
{
"id": "noteid",
"noteType": "",
"patientId": "patient_id",
"text": "Example text",
"note_name": "dataset/awesome-dataset/fhirStores/awesome-fhirstore/fhir/Note/noteid"
}
"""
configuration = datanode.Configuration(host=host)
offset = 0
limit = 10
with datanode.ApiClient(configuration) as api_client:
note_instance = note_api.NoteApi(api_client)
# Obtain all clinical notes
next_page = True
while next_page:
notes = note_instance.list_notes(dataset_id,
fhir_store_id,
offset=offset,
limit=limit)
# change from snake case to camel case
sanitized_notes = api_client.sanitize_for_serialization(
notes.notes)
for note in sanitized_notes:
# note_name is added for convenience for the controller as
# it is needed to store the annotations
note[
"note_name"] = f"dataset/{dataset_id}/fhirStores/{fhir_store_id}/fhir/Note/{note['identifier']}"
yield note
next_page = notes.links.next
offset += limit
def store_annotations(host: str,
dataset_id: str,
annotation_store_id: str,
annotations: dict,
delete_existing_annotations: bool = True):
"""Store submission annotated notes. Delete an annotation store if
the annotation store exists, then create a new annotation store, then
store the annotation.
Args:
host: Data node host IP
dataset_id: Dataset Id
annotation_store_id: Annotation store Id
annotations: Data Node Annotations
delete_existing_annotations: To delete existing annotation store.
Default is True.
"""
configuration = datanode.Configuration(host=host)
with datanode.ApiClient(configuration) as api_client:
annot_store_instance = annotation_store_api.AnnotationStoreApi(
api_client)
try:
# Always try to delete the annotation store prior to
# storing predictions
if delete_existing_annotations:
annot_store_instance.delete_annotation_store(
dataset_id, annotation_store_id)
print("Deleted existing Annotation Store")
except datanode.rest.ApiException:
pass
try:
annot_store_instance.create_annotation_store(dataset_id,
annotation_store_id,
body={})
print("Created Annotation Store")
except datanode.rest.ApiException:
print("Using existing Annotation Store")
for annotation in annotations:
annotation_id = annotation['annotationSource']['resourceSource'][
'name'].split("/")[-1]
_store_annotation(host=host,
dataset_id=dataset_id,
annotation_store_id=annotation_store_id,
annotation_id=annotation_id,
annotation=annotation)
def get_annotation_store(host: str,
dataset_id: str,
annotation_store_id: str,
create_if_missing: bool = False) -> AnnotationStore:
"""Creates an annotation store
Args:
host: Data node host IP
dataset_id: Dataset Id
annotation_store_id: Annotation store Id
create_if_missing: Creates annotation store if the resource
doesn't exist
Returns:
Data node Annotation Store object
Examples:
>>> annotation = create_annotation_store(
>>> host="0.0.0.0/api/v1", dataset_id="awesome-dataset",
>>> annotation_store_id="awesome-annotation-store"
>>> )
"""
configuration = datanode.Configuration(host=host)
with datanode.ApiClient(configuration) as api_client:
annot_store_instance = annotation_store_api.AnnotationStoreApi(
api_client)
try:
# get the annotation store
annotation_store_obj = annot_store_instance.get_annotation_store(
dataset_id, annotation_store_id)
except datanode.rest.ApiException as err:
if err.status == 404 and create_if_missing:
annotation_store_obj = annot_store_instance.create_annotation_store(
dataset_id, annotation_store_id, body={})
else:
raise err
return annotation_store_obj
import json
import sys
import synapseclient
import datanode
import datanode.apis
import datanode.models
from datanode.rest import ApiException
import nlpsandboxclient.utils
syn = synapseclient.login()
host = ""
if host == "":
raise ValueError("Must set host to be data node URL")
configuration = datanode.Configuration(host=host)
dataset_id = '2014-i2b2-20201203'
fhir_store_id = 'evaluation'
annotation_store_id = 'goldstandard'
# Get evaluation-patient-bundles.json
# Version 4 for v1.0.1 schemas
# Version 5 for v1.0.2 schemas
json_ent = syn.get("syn23593068", version=6)
json_filename = json_ent.path
def get_or_create_resource(get_func, create_func, *args, **kwargs):
"""Get or create a data node resource
Args: