def write_annotation(dataset: str,
annotation: Annotation,
user: User,
move_key: str = "") -> str:
if annotation.user is None:
annotation.user = user.email
authorized = (annotation.user == user.email and user.can_write_own(dataset)) or \
(annotation.user != user.email and user.can_write_others(dataset))
if not authorized:
raise HTTPException(
status_code=401,
detail=
f"no permission to add annotation for user {annotation.user} on dataset {dataset}"
)
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_V2, dataset, annotation.user])
annotation_json = jsonable_encoder(annotation, exclude_unset=True)
key = annotation.key()
collection.document(key).set(annotation_json)
if move_key != "" and move_key != key:
collection.document(move_key).delete()
return key
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in put annotation for dataset {dataset}")
def delete_annotation(dataset: str,
key: str,
user_email: str = "",
user: User = Depends(get_user)):
""" Delete an annotation based on the key supplied in the URL. If the client has proper authorization,
annotations for another user can be deleted by supplying the query string '[email protected]'.
"""
if user_email == "":
user_email = user.email
authorized = (user_email == user.email and user.can_write_own(dataset)) or \
(user_email != user.email and user.can_write_others(dataset))
if not authorized:
raise HTTPException(
status_code=401,
detail=
f"no permission to delete annotation for user {user_email} on dataset {dataset}"
)
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_V2, dataset, user_email])
collection.document(key).delete()
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=
f"error in deleting annotation with key {key} for dataset {dataset}"
)
def get_annotations(dataset: str,
groups: str = "",
user: str = "",
requestor: User = Depends(get_user)):
""" Returns all annotations for the user defined by the accompanying Authorization token
or the query string user if the requestor has admin permissions.
Return format is JSON list with annotations and generated fields "user" and "key",
where "key" is a user-scoped key used in DELETE requests for annotations.
Optional query string "groups" (names separated by commas) can result in larger set
of annotations returned, corresponding to all annotations for the given groups.
The groups can only be ones in which the user is a member unless user has admin permissions.
"""
if not requestor.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
if user == "":
user = requestor.email
elif user != requestor.email and not requestor.is_admin():
raise HTTPException(
status_code=401,
detail=f"get of user {user} requires admin permissions")
output = []
members = set([user])
if groups != "":
groups_queried = set(groups.split(','))
if len(groups_queried) > 0:
if requestor.is_admin():
groups_added = groups_queried
else:
groups_added = groups_queried.intersection(requestor.groups)
if len(groups_added) == 0:
raise HTTPException(
status_code=400,
detail=
f"requestor {requestor.email} is not member of requested groups {groups_queried}"
)
members.update(group_members(requestor, groups_added))
try:
for member in members:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_V2, dataset, member])
annotations_ref = collection.get()
for annotation_ref in annotations_ref:
annotation_dict = annotation_ref.to_dict()
annotation_dict["user"] = member
annotation_dict["key"] = annotation_ref.id
output.append(annotation_dict)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
return output
def pull_request(user_email: str, user: User = Depends(get_user)):
"""Returns user's pull requests. If "user_email" query string is provided, that user's pull
request is returned if the requester has sufficient permission. All pull requests can
be obtained by using the query string "user_email=all" if the user is admin.
{"kv/scope1/key1": value1, "kv/scope2/key2": value2}
"""
prs = {}
try:
collection = firestore.get_collection([CLIO_PULL_REQUESTS])
emails = set()
if user_email == "":
emails = set([user.email])
elif user_email == "all":
if not user.is_admin():
raise HTTPException(
status_code=401,
detail=f"no permission to access all pull requests")
emails = [ref.id for ref in collection.list_documents()]
else:
authorized = (user_email == user.email or user.is_admin())
if not authorized:
raise HTTPException(
status_code=401,
detail=
f"no permission to access pull requests for user {user_email}"
)
emails = set([user_email])
for email in emails:
user_ref = collection.document(email).get()
user_prs = {}
pr_types = user_ref.reference.collections()
for pr_type in pr_types:
user_prs[pr_type.id] = {}
for pr in pr_type.stream():
user_prs[pr_type.id][pr.id] = pr.to_dict()
if len(user_prs) != 0:
prs[email] = user_prs
except HTTPException as e:
raise e
except Exception as e:
print(e)
raise HTTPException(status_code=HTTPStatus.BAD_REQUEST,
detail=f"error attempting pull request: {e}")
return prs
def get_tag_to_uuid(dataset: str,
annotation_type: str,
tag: str,
user: User = Depends(get_user)):
""" Returns the corresponding dvid UUID of the given tag for the given scope.
Returns:
A string of the uuid corresponding to the tag, e.g., "74ea83"
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
tag_to_uuid = cache.get_value(collection_path=[CLIO_ANNOTATIONS_GLOBAL],
document='metadata',
path=['neurons', 'VNC', 'tag_to_uuid'])
if not tag_to_uuid:
raise HTTPException(
status_code=404,
detail=
f"Could not find any tag_to_uuid for annotation type {annotation_type} in dataset {dataset}"
)
if tag not in tag_to_uuid:
raise HTTPException(
status_code=404,
detail=
f"Could not find tag {tag} for annotation type {annotation_type} in dataset {dataset}"
)
return tag_to_uuid[tag]
def searches(dataset: str,
x: int,
y: int,
z: int,
payload: dict,
user: User = Depends(get_user)):
if not user.has_role("clio_general", dataset):
raise HTTPException(
status_code=401,
detail=
f"user does not have permission to write searches to dataset {dataset_id}"
)
try:
# we only allow one annotation per email and location key so if it exists, replace.
payload["timestamp"] = time.time()
payload["dataset"] = dataset
payload["location"] = [x, y, z]
payload["locationkey"] = f"{x}_{y}_{z}"
payload["email"] = user.email
collection = firestore.get_collection(
[CLIO_SAVEDSEARCHES, "USER", "searches"])
collection.document().set(payload)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=
f"error in put annotation ({x},{y},{z}) for dataset {dataset}")
def delete_searches(dataset: str,
x: int,
y: int,
z: int,
user: User = Depends(get_user)):
if not user.has_role("clio_general", dataset):
raise HTTPException(
status_code=401,
detail=
f"user does not have permission to delete searches in dataset {dataset_id}"
)
try:
# delete only supported from interface
# (delete by dataset + user name + xyz)
collection = firestore.get_collection(
[CLIO_SAVEDSEARCHES, "USER", "searches"])
match_list = collection.where("email", "==", user.email).where(
"locationkey", "==", f"{x}_{y}_{z}").where("dataset", "==",
dataset).get()
for match in match_list:
match.reference.delete()
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in deleting saved searches for dataset {dataset}")
def delete_atlas(dataset: str,
x: int,
y: int,
z: int,
user: User = Depends(get_user)):
if not user.can_write_own(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to delete annotations in dataset {dataset}")
try:
# delete only supported from interface
# (delete by dataset + user name + xyz)
collection = firestore.get_collection(
[CLIO_ANNOTATIONS, "ATLAS", "annotations"])
match_list = collection.where("user", "==", user.email).where(
"locationkey", "==", f"{x}_{y}_{z}").where("dataset", "==",
dataset).get()
for match in match_list:
match.reference.delete()
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=
f"error in deleting annotation ({x},{y},{z}) for dataset {dataset}"
)
def post_users(postdata: Dict[str, UserPayload],
user: User = Depends(get_user)):
if not user.is_admin():
raise HTTPException(status_code=401,
detail="user lacks permission for /users endpoint")
try:
collection = firestore.get_collection([CLIO_USERS])
for email, data in postdata.items():
user_dict = data.dict()
collection.document(email).set(user_dict)
user_dict["email"] = email
user_with_email = User(**user_dict)
users.cache_user(user_with_email)
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail=f"error in posting users: {e}")
def get_atlas(dataset: str, user: User = Depends(get_user)):
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS, "ATLAS", "annotations"])
if dataset != "all":
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations")
annotations = collection.where("user", "==", user.email).where(
"dataset", "==", dataset).get()
output = {}
for annotation in annotations:
res = annotation.to_dict()
res["id"] = annotation.id
output[res["locationkey"]] = res
return output
else:
annotations = collection.stream()
output = []
for annotation in annotations:
res = annotation.to_dict()
annot_dataset = res.get("dataset", "")
if user.can_read(annot_dataset):
if "verified" not in res:
print(f"bad atlas point, adding 'verified': {res}")
res["verified"] = False
annotation.reference.set(res)
res["id"] = annotation.id
if res["verified"] or res["user"] == user.email:
output.append(res)
elif user.can_write_others(annot_dataset):
output.append(res)
return output
except HTTPException as e:
raise e
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving atlas for dataset {dataset}: {e}")
return
def get_users(user: User = Depends(get_user)) -> Dict[str, User]:
if not user.is_admin():
raise HTTPException(status_code=401,
detail="user lacks permission for /users endpoint")
try:
return users.refresh_cache()
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail=f"error in retrieving users: {e}")
def get_signature(user: User, dataset: str, point: tuple):
if not user.has_role("clio_general", dataset):
raise HTTPException(status_code=403, detail="user doesn't have authorization")
try:
pt, sig = fetch_signature(dataset, *point)
res = {"point": pt, "signature": str(sig)}
except Exception as e:
res = {"messsage": str(e)}
return res
def get_all_annotations(dataset: str,
annotation_type: str,
cursor: str = None,
size: int = MAX_ANNOTATIONS_RETURNED,
user: User = Depends(get_user)):
""" Returns all current neuron annotations for the given dataset and annotation type.
Query strings:
cursor (str): If supplied, annotations after the given id field are sent.
size (int): If supplied, at most this many annotations are returned.
Returns:
A JSON list of the annotations.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
if cursor:
cursor = f'id{cursor}' # convert id into key format
output = []
try:
t0 = time.time()
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_GLOBAL, annotation_type,
dataset]).where('_head', '==', True)
pagesize = min(size, 5000)
while True:
query = collection.limit(pagesize).order_by('__name__')
if cursor:
query = query.start_after({"__name__": cursor})
t1 = time.time()
retrieved = 0
for snapshot in query.stream():
retrieved += 1
annotation = remove_reserved_fields(snapshot.to_dict())
output.append(annotation)
cursor = snapshot.id
print(f'processed {retrieved} in {time.time() - t1} secs')
if retrieved < pagesize or len(output) == size:
break
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
print(f'{len(output)} total processed in {time.time() - t0} secs')
return output
def get_matches(user: User, dataset: str, point: tuple):
if not user.has_role("clio_general", dataset):
raise HTTPException(status_code=403, detail="user doesn't have authorization")
try:
data = find_similar_signatures(dataset, *point)
res = {"matches": data}
if len(data) == 0:
res["message"] = "no matches"
except Exception as e:
res = {"messsage": str(e)}
return res
def delete_users(deleted_emails: List, user: User = Depends(get_user)):
if not user.is_admin():
raise HTTPException(status_code=401,
detail="user lacks permission for /users endpoint")
try:
collection = firestore.get_collection([CLIO_USERS])
for email in deleted_emails:
collection.document(email).delete()
users.uncache_user(email)
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail=f"error in deleting users: {e}")
def get_volume(volume: str, current_user: User = Depends(get_user)):
global cache
try:
if public_dataset(volume) or current_user.can_read(volume):
doc_ref = firestore.get_collection(CLIO_VOLUMES).document(
volume).get()
if doc_ref.exists:
data = doc_ref.to_dict()
cache[volume] = Volume(**data)
return data
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving volumes' metadata")
def edit_subvol(dataset: str,
subvol: SubVolume,
user: User = Depends(get_user)):
"""
Creates a neuroglancer precomputed subvolume cutout given bounding box. Returns
python script that can be downloaded and used to open and then submit change to Clio store.
"""
if not user.can_read(dataset) or not user.can_write_own(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to edit subvolume in dataset {dataset}")
try:
dataset_obj = get_dataset(dataset)
# cv = CloudVolume(dataset_obj.location)
# vol_dest = subvol.destination(user.email)
# vol_bounds = subvol.bbox()
# cv.transfer_to(vol_dest, vol_bounds)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error editing subvolume for dataset {dataset}: {e}")
def get_volumes(current_user: User = Depends(get_user)):
global cache
try:
collection = firestore.get_collection(CLIO_VOLUMES)
volumes_out = {}
for volume in collection.stream():
volume_info = volume.to_dict()
cache[volume.id] = Volume(**volume_info)
if public_dataset(volume.id) or current_user.can_read(volume.id):
volumes_out[volume.id] = volume_info
return volumes_out
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving volumes' metadata")
def delete_datasets(to_delete: List[str],
current_user: User = Depends(get_user)):
if not current_user.is_admin():
raise HTTPException(
status_code=401,
detail="user must be admin to delete dataset metadata")
try:
collection = firestore.get_collection(CLIO_DATASETS)
for dataset_id in to_delete:
collection.document(dataset_id).delete()
# TODO -- Allow deletion of all data corresponding to this dataset?
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail=f"error in deleting datasets {to_delete}")
def post_volumes(volumes: Dict[str, Volume],
current_user: User = Depends(get_user)):
global cache
if not current_user.is_admin():
raise HTTPException(status_code=401,
detail="user must be admin to set volume metadata")
try:
collection = firestore.get_collection(CLIO_VOLUMES)
for volume_id, volume in volumes.items():
data = volume.dict(exclude_unset=True)
collection.document(volume_id).set(data)
cache[volume_id] = Volume(**data)
except Exception as e:
print(e)
raise HTTPException(status_code=400, detail="error in POSTing volumes")
def get_annotations(dataset: str,
annotation_type: str,
id: str,
version: str = "",
changes: bool = False,
id_field: str = "bodyid",
user: User = Depends(get_user)):
""" Returns the neuron annotation associated with the given id.
Query strings:
version (str): If supplied, annotations are for the given dataset version.
changes (bool): If True, returns list of changes to this annotation across all versions.
id_field (str): The id field name (default: "bodyid")
Returns:
A JSON list (if changes requested or multiple ids given) or JSON object if not.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
if version != "":
cur_dataset = get_dataset(dataset)
if cur_dataset.tag == version:
version = ""
if "," in id:
id_strs = id.split(",")
ids = [int(id_str) for id_str in id_strs]
else:
ids = [int(id)]
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_GLOBAL, annotation_type, dataset])
return run_query_on_ids(collection, collection, ids, id_field, version,
changes)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
def get_dataset(dataset: str,
templates: bool = False,
current_user: User = Depends(get_user)):
try:
if public_dataset(dataset) or current_user.can_read(dataset):
doc_ref = firestore.get_collection(CLIO_DATASETS).document(
dataset).get()
if not doc_ref.exists:
raise Exception(f'could not find dataset {dataset}')
if templates:
return doc_ref.to_dict()
else:
return replace_templates(doc_ref.to_dict())
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving datasets' metadata")
def get_uuid_to_tag(dataset: str,
annotation_type: str,
uuid: str,
user: User = Depends(get_user)):
""" Returns the corresponding string tag for the given dvid UUID for the given scope.
Returns:
A string of the tag corresponding to the uuid, e.g., "v0.3.32"
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
uuid_to_tag = cache.get_value(collection_path=[CLIO_ANNOTATIONS_GLOBAL],
document='metadata',
path=['neurons', 'VNC', 'uuid_to_tag'])
if not uuid_to_tag:
raise HTTPException(
status_code=404,
detail=
f"Could not find any uuid_to_tag for annotation type {annotation_type} in dataset {dataset}"
)
found_tag = None
num_found = 0
for stored_uuid in uuid_to_tag:
if len(stored_uuid) < len(uuid) and uuid.startswith(stored_uuid):
num_found += 1
found_tag = uuid_to_tag[stored_uuid]
if len(stored_uuid) >= len(uuid) and stored_uuid.startswith(uuid):
num_found += 1
found_tag = uuid_to_tag[stored_uuid]
if num_found > 1:
raise HTTPException(
status_code=400,
detail=
f"uuid {uuid} is ambiguous because more than one hit for annotation type {annotation_type} in dataset {dataset}"
)
if not found_tag:
raise HTTPException(
status_code=404,
detail=
f"Could not find uuid {uuid} for annotation type {annotation_type} in dataset {dataset}"
)
return found_tag
def submit_subvol(dataset: str, payload: dict, user: User = Depends(get_user)):
"""
Stores reference to an edited neuroglancer subvolume.
"""
if not user.can_write_own(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to submit subvolume in dataset {dataset}")
try:
payload["_timestamp"] = time.time()
collection = firestore.get_collection(
[CLIO_SUBVOL, user.email, "submissions"])
collection.document().set(payload)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error submitting subvolume for dataset {dataset}: {e}")
def delete_volumes(to_delete: List[str],
current_user: User = Depends(get_user)):
global cache
if not current_user.is_admin():
raise HTTPException(
status_code=401,
detail="user must be admin to delete volume metadata")
try:
collection = firestore.get_collection(CLIO_VOLUMES)
for volume_id in to_delete:
collection.document(volume_id).delete()
if volume_id in cache:
del cache[volume_id]
# TODO -- Allow deletion of all data corresponding to this volume?
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail=f"error in deleting volumes {to_delete}")
async def post_neuprint_custom(dataset: str, payload: NeuprintRequest, user: User = Depends(get_user)):
if not user.has_role("clio_general", dataset):
raise HTTPException(status_code=403, detail="user doesn't have authorization for this dataset")
cur_dataset = get_dataset(dataset)
if cur_dataset.neuprintHTTP:
neuprint_server = cur_dataset.neuprintHTTP.server
else:
raise HTTPException(status_code=400, detail=f"dataset {dataset} has no assigned neuprint server")
try:
async with client_session.post(f'https://{neuprint_server}/api/custom/custom', data=payload.json(), headers=neuprint_headers) as resp:
response = await resp.json()
except Exception as e:
print(e)
raise HTTPException(status_code=400, detail=f"error in neuprint request for dataset {dataset}")
return response
def get_annotations(dataset: str,
groups: str = "",
user: User = Depends(get_user)):
""" Returns all annotations for the user defined by the accompanying Authorization token.
Return format is JSON object with annotations as enclosed key-value pairs. Keys are
used in move and delete operations.
Optional query string "groups" (names separated by commas) can result in larger set
of annotations returned, corresponding to all annotations for the given groups.
The groups can only be ones in which the user is a member.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
output = {}
members = set([user.email])
if groups != "":
groups_queried = set(groups.split(','))
if len(groups_queried) > 0:
groups_added = groups_queried.intersection(user.groups)
if len(groups_added) == 0:
raise HTTPException(
status_code=400,
detail=
f"user {user.email} is not member of requested groups {groups_queried}"
)
members.update(group_members(user, groups_added))
try:
for member in members:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_V2, dataset, member])
annotations_ref = collection.get()
for annotation_ref in annotations_ref:
annotation_dict = annotation_ref.to_dict()
output[annotation_ref.id] = annotation_dict
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
return output
def post_datasets(datasets: Dict[str, Dataset],
current_user: User = Depends(get_user)):
if not current_user.is_admin():
raise HTTPException(
status_code=401,
detail="user must be admin to set dataset metadata")
try:
collection = firestore.get_collection(CLIO_DATASETS)
for dataset_id, dataset in datasets.items():
cached_dataset = get_cached_dataset(dataset_id)
if cached_dataset and cached_dataset.tag and cached_dataset.tag > dataset.tag:
raise Exception(
f'posted dataset {dataset_id} has tag {dataset.tag} earlier than current {cached_dataset.tag}'
)
collection.document(dataset_id).set(
dataset.dict(exclude_unset=True))
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in POSTing datasets: {e}")
def get_datasets(templates: bool = False,
current_user: User = Depends(get_user)):
try:
collection = firestore.get_collection(CLIO_DATASETS)
t0 = time.time()
datasets_out = {}
for dataset in collection.stream():
dataset_info = dataset.to_dict()
if public_dataset(dataset.id) or current_user.can_read(dataset.id):
if templates:
datasets_out[dataset.id] = dataset_info
else:
datasets_out[dataset.id] = replace_templates(dataset_info)
print(
f'Retrieved {len(datasets_out)} datasets in {time.time() - t0} seconds'
)
return datasets_out
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving datasets' metadata")
def get_searches(dataset: str, user: User = Depends(get_user)):
if not user.has_role("clio_general", dataset):
raise HTTPException(
status_code=401,
detail=f"user does not have permission to read dataset {dataset_id}"
)
try:
collection = firestore.get_collection(
[CLIO_SAVEDSEARCHES, "USER", "searches"])
searches = collection.where("email", "==",
user.email).where("dataset", "==",
dataset).get()
output = {}
for search in searches:
res = search.to_dict()
res["id"] = search.id
output[res["locationkey"]] = res
return output
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in getting saved searches for dataset {dataset}")