def get_tag_to_uuid(dataset: str,
annotation_type: str,
tag: str,
user: User = Depends(get_user)):
""" Returns the corresponding dvid UUID of the given tag for the given scope.
Returns:
A string of the uuid corresponding to the tag, e.g., "74ea83"
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
tag_to_uuid = cache.get_value(collection_path=[CLIO_ANNOTATIONS_GLOBAL],
document='metadata',
path=['neurons', 'VNC', 'tag_to_uuid'])
if not tag_to_uuid:
raise HTTPException(
status_code=404,
detail=
f"Could not find any tag_to_uuid for annotation type {annotation_type} in dataset {dataset}"
)
if tag not in tag_to_uuid:
raise HTTPException(
status_code=404,
detail=
f"Could not find tag {tag} for annotation type {annotation_type} in dataset {dataset}"
)
return tag_to_uuid[tag]
def get_annotations(dataset: str,
groups: str = "",
user: str = "",
requestor: User = Depends(get_user)):
""" Returns all annotations for the user defined by the accompanying Authorization token
or the query string user if the requestor has admin permissions.
Return format is JSON list with annotations and generated fields "user" and "key",
where "key" is a user-scoped key used in DELETE requests for annotations.
Optional query string "groups" (names separated by commas) can result in larger set
of annotations returned, corresponding to all annotations for the given groups.
The groups can only be ones in which the user is a member unless user has admin permissions.
"""
if not requestor.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
if user == "":
user = requestor.email
elif user != requestor.email and not requestor.is_admin():
raise HTTPException(
status_code=401,
detail=f"get of user {user} requires admin permissions")
output = []
members = set([user])
if groups != "":
groups_queried = set(groups.split(','))
if len(groups_queried) > 0:
if requestor.is_admin():
groups_added = groups_queried
else:
groups_added = groups_queried.intersection(requestor.groups)
if len(groups_added) == 0:
raise HTTPException(
status_code=400,
detail=
f"requestor {requestor.email} is not member of requested groups {groups_queried}"
)
members.update(group_members(requestor, groups_added))
try:
for member in members:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_V2, dataset, member])
annotations_ref = collection.get()
for annotation_ref in annotations_ref:
annotation_dict = annotation_ref.to_dict()
annotation_dict["user"] = member
annotation_dict["key"] = annotation_ref.id
output.append(annotation_dict)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
return output
def get_atlas(dataset: str, user: User = Depends(get_user)):
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS, "ATLAS", "annotations"])
if dataset != "all":
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations")
annotations = collection.where("user", "==", user.email).where(
"dataset", "==", dataset).get()
output = {}
for annotation in annotations:
res = annotation.to_dict()
res["id"] = annotation.id
output[res["locationkey"]] = res
return output
else:
annotations = collection.stream()
output = []
for annotation in annotations:
res = annotation.to_dict()
annot_dataset = res.get("dataset", "")
if user.can_read(annot_dataset):
if "verified" not in res:
print(f"bad atlas point, adding 'verified': {res}")
res["verified"] = False
annotation.reference.set(res)
res["id"] = annotation.id
if res["verified"] or res["user"] == user.email:
output.append(res)
elif user.can_write_others(annot_dataset):
output.append(res)
return output
except HTTPException as e:
raise e
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving atlas for dataset {dataset}: {e}")
return
def get_all_annotations(dataset: str,
annotation_type: str,
cursor: str = None,
size: int = MAX_ANNOTATIONS_RETURNED,
user: User = Depends(get_user)):
""" Returns all current neuron annotations for the given dataset and annotation type.
Query strings:
cursor (str): If supplied, annotations after the given id field are sent.
size (int): If supplied, at most this many annotations are returned.
Returns:
A JSON list of the annotations.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
if cursor:
cursor = f'id{cursor}' # convert id into key format
output = []
try:
t0 = time.time()
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_GLOBAL, annotation_type,
dataset]).where('_head', '==', True)
pagesize = min(size, 5000)
while True:
query = collection.limit(pagesize).order_by('__name__')
if cursor:
query = query.start_after({"__name__": cursor})
t1 = time.time()
retrieved = 0
for snapshot in query.stream():
retrieved += 1
annotation = remove_reserved_fields(snapshot.to_dict())
output.append(annotation)
cursor = snapshot.id
print(f'processed {retrieved} in {time.time() - t1} secs')
if retrieved < pagesize or len(output) == size:
break
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
print(f'{len(output)} total processed in {time.time() - t0} secs')
return output
def get_volume(volume: str, current_user: User = Depends(get_user)):
global cache
try:
if public_dataset(volume) or current_user.can_read(volume):
doc_ref = firestore.get_collection(CLIO_VOLUMES).document(
volume).get()
if doc_ref.exists:
data = doc_ref.to_dict()
cache[volume] = Volume(**data)
return data
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving volumes' metadata")
def get_volumes(current_user: User = Depends(get_user)):
global cache
try:
collection = firestore.get_collection(CLIO_VOLUMES)
volumes_out = {}
for volume in collection.stream():
volume_info = volume.to_dict()
cache[volume.id] = Volume(**volume_info)
if public_dataset(volume.id) or current_user.can_read(volume.id):
volumes_out[volume.id] = volume_info
return volumes_out
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving volumes' metadata")
def get_annotations(dataset: str,
annotation_type: str,
id: str,
version: str = "",
changes: bool = False,
id_field: str = "bodyid",
user: User = Depends(get_user)):
""" Returns the neuron annotation associated with the given id.
Query strings:
version (str): If supplied, annotations are for the given dataset version.
changes (bool): If True, returns list of changes to this annotation across all versions.
id_field (str): The id field name (default: "bodyid")
Returns:
A JSON list (if changes requested or multiple ids given) or JSON object if not.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
if version != "":
cur_dataset = get_dataset(dataset)
if cur_dataset.tag == version:
version = ""
if "," in id:
id_strs = id.split(",")
ids = [int(id_str) for id_str in id_strs]
else:
ids = [int(id)]
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_GLOBAL, annotation_type, dataset])
return run_query_on_ids(collection, collection, ids, id_field, version,
changes)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
def get_uuid_to_tag(dataset: str,
annotation_type: str,
uuid: str,
user: User = Depends(get_user)):
""" Returns the corresponding string tag for the given dvid UUID for the given scope.
Returns:
A string of the tag corresponding to the uuid, e.g., "v0.3.32"
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
uuid_to_tag = cache.get_value(collection_path=[CLIO_ANNOTATIONS_GLOBAL],
document='metadata',
path=['neurons', 'VNC', 'uuid_to_tag'])
if not uuid_to_tag:
raise HTTPException(
status_code=404,
detail=
f"Could not find any uuid_to_tag for annotation type {annotation_type} in dataset {dataset}"
)
found_tag = None
num_found = 0
for stored_uuid in uuid_to_tag:
if len(stored_uuid) < len(uuid) and uuid.startswith(stored_uuid):
num_found += 1
found_tag = uuid_to_tag[stored_uuid]
if len(stored_uuid) >= len(uuid) and stored_uuid.startswith(uuid):
num_found += 1
found_tag = uuid_to_tag[stored_uuid]
if num_found > 1:
raise HTTPException(
status_code=400,
detail=
f"uuid {uuid} is ambiguous because more than one hit for annotation type {annotation_type} in dataset {dataset}"
)
if not found_tag:
raise HTTPException(
status_code=404,
detail=
f"Could not find uuid {uuid} for annotation type {annotation_type} in dataset {dataset}"
)
return found_tag
def get_dataset(dataset: str,
templates: bool = False,
current_user: User = Depends(get_user)):
try:
if public_dataset(dataset) or current_user.can_read(dataset):
doc_ref = firestore.get_collection(CLIO_DATASETS).document(
dataset).get()
if not doc_ref.exists:
raise Exception(f'could not find dataset {dataset}')
if templates:
return doc_ref.to_dict()
else:
return replace_templates(doc_ref.to_dict())
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving datasets' metadata")
def get_annotations(dataset: str,
groups: str = "",
user: User = Depends(get_user)):
""" Returns all annotations for the user defined by the accompanying Authorization token.
Return format is JSON object with annotations as enclosed key-value pairs. Keys are
used in move and delete operations.
Optional query string "groups" (names separated by commas) can result in larger set
of annotations returned, corresponding to all annotations for the given groups.
The groups can only be ones in which the user is a member.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
output = {}
members = set([user.email])
if groups != "":
groups_queried = set(groups.split(','))
if len(groups_queried) > 0:
groups_added = groups_queried.intersection(user.groups)
if len(groups_added) == 0:
raise HTTPException(
status_code=400,
detail=
f"user {user.email} is not member of requested groups {groups_queried}"
)
members.update(group_members(user, groups_added))
try:
for member in members:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_V2, dataset, member])
annotations_ref = collection.get()
for annotation_ref in annotations_ref:
annotation_dict = annotation_ref.to_dict()
output[annotation_ref.id] = annotation_dict
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
return output
def get_datasets(templates: bool = False,
current_user: User = Depends(get_user)):
try:
collection = firestore.get_collection(CLIO_DATASETS)
t0 = time.time()
datasets_out = {}
for dataset in collection.stream():
dataset_info = dataset.to_dict()
if public_dataset(dataset.id) or current_user.can_read(dataset.id):
if templates:
datasets_out[dataset.id] = dataset_info
else:
datasets_out[dataset.id] = replace_templates(dataset_info)
print(
f'Retrieved {len(datasets_out)} datasets in {time.time() - t0} seconds'
)
return datasets_out
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving datasets' metadata")
def edit_subvol(dataset: str,
subvol: SubVolume,
user: User = Depends(get_user)):
"""
Creates a neuroglancer precomputed subvolume cutout given bounding box. Returns
python script that can be downloaded and used to open and then submit change to Clio store.
"""
if not user.can_read(dataset) or not user.can_write_own(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to edit subvolume in dataset {dataset}")
try:
dataset_obj = get_dataset(dataset)
# cv = CloudVolume(dataset_obj.location)
# vol_dest = subvol.destination(user.email)
# vol_bounds = subvol.bbox()
# cv.transfer_to(vol_dest, vol_bounds)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error editing subvolume for dataset {dataset}: {e}")
def get_head_uuid(dataset: str,
annotation_type: str,
user: User = Depends(get_user)):
""" Returns the head version uuid for the given scope.
Returns:
A string of the HEAD version uuid, e.g., "74ea83"
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
head_uuid = cache.get_value(collection_path=[CLIO_ANNOTATIONS_GLOBAL],
document='metadata',
path=['neurons', 'VNC', 'head_uuid'])
if not head_uuid:
raise HTTPException(
status_code=404,
detail=
f"Could not find any head_uuid for annotation type {annotation_type} in dataset {dataset}"
)
return head_uuid
def get_versions(dataset: str,
annotation_type: str,
user: User = Depends(get_user)):
""" Returns the versions for the given scope.
Returns:
A dict with tag keys and corresponding dvid UUIDs as value.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
tag_to_uuid = cache.get_value(collection_path=[CLIO_ANNOTATIONS_GLOBAL],
document='metadata',
path=['neurons', 'VNC', 'tag_to_uuid'])
if not tag_to_uuid:
raise HTTPException(
status_code=404,
detail=
f"Could not find any tag_to_uuid for annotation type {annotation_type} in dataset {dataset}"
)
return tag_to_uuid
def get_fields(dataset: str,
annotation_type: str,
user: User = Depends(get_user)):
""" Returns all fields within annotations for the given scope.
Returns:
A JSON list of the fields present in at least one annotation.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
fields = cache.get_value(collection_path=[CLIO_ANNOTATIONS_GLOBAL],
document='metadata',
path=['neurons', 'VNC', 'fields'])
if not fields:
raise HTTPException(
status_code=404,
detail=
f"Could not find any fields for annotation type {annotation_type} in dataset {dataset}"
)
return fields
def get_annotations(dataset: str, annotation_type: str, query: Union[List[Dict], Dict], version: str = "", changes: bool = False, \
id_field: str = "bodyid", onlyid: bool = False, user: User = Depends(get_user)):
""" Executes a query on the annotations using supplied JSON.
The JSON query format uses field names as the keys, and desired values.
Example:
{ "bodyid": 23, "hemilineage": "0B", ... }
Each field value must be true, i.e., the conditions or ANDed together.
If a list of queries (JSON object per query) is POSTed, the results for each query or ORed
together with duplicate annotations removed.
Query strings:
version (str): If supplied, annotations are for the given dataset version.
changes (bool): If True, returns list of changes to this annotation across all versions.
id_field (str): The id field name (default: "bodyid") that should be integers.
onlyid (bool): If true (false by default), will only return a list of id field values that match.
Returns:
A JSON list of objects.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
if version != "":
cur_dataset = get_dataset(dataset)
if cur_dataset.tag == version:
version = ""
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_GLOBAL, annotation_type, dataset])
results = []
if isinstance(query, dict):
query = [query]
query_num = 1
for cur_query in query:
nonid_query = collection
ids = []
for key in cur_query:
if key == id_field:
if isinstance(cur_query[key], int):
ids = [cur_query[key]]
elif isinstance(cur_query[key], list):
ids = cur_query[key]
else:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=
f"id field must be int or list of ints, got: {cur_query[key]}"
)
continue
else:
if key in set_fields:
op = "array_contains"
elif isinstance(cur_query[key], list):
if len(cur_query[key]) > 10:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=
f"currently no more than 10 values can be queried at a time"
)
if len(
cur_query[key]
) == 1: # counters apparent issue with using 'in'. TODO: determine underlying issue.
op = "=="
cur_query[key] = cur_query[key][0]
else:
op = "in"
else:
op = "=="
nonid_query = nonid_query.where(key, op, cur_query[key])
if len(ids) == 0:
cur_results = run_query(collection, nonid_query, id_field,
version, changes, onlyid)
else:
cur_results = run_query_on_ids(collection, nonid_query, ids,
id_field, version, changes,
onlyid)
if query_num > 1: # Or these results into previous
merge_annotations(results, cur_results, id_field)
else:
results = cur_results
query_num += 1
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
return results