def searches(dataset: str,
x: int,
y: int,
z: int,
payload: dict,
user: User = Depends(get_user)):
if not user.has_role("clio_general", dataset):
raise HTTPException(
status_code=401,
detail=
f"user does not have permission to write searches to dataset {dataset_id}"
)
try:
# we only allow one annotation per email and location key so if it exists, replace.
payload["timestamp"] = time.time()
payload["dataset"] = dataset
payload["location"] = [x, y, z]
payload["locationkey"] = f"{x}_{y}_{z}"
payload["email"] = user.email
collection = firestore.get_collection(
[CLIO_SAVEDSEARCHES, "USER", "searches"])
collection.document().set(payload)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=
f"error in put annotation ({x},{y},{z}) for dataset {dataset}")
def write_annotation(dataset: str,
annotation: Annotation,
user: User,
move_key: str = "") -> str:
if annotation.user is None:
annotation.user = user.email
authorized = (annotation.user == user.email and user.can_write_own(dataset)) or \
(annotation.user != user.email and user.can_write_others(dataset))
if not authorized:
raise HTTPException(
status_code=401,
detail=
f"no permission to add annotation for user {annotation.user} on dataset {dataset}"
)
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_V2, dataset, annotation.user])
annotation_json = jsonable_encoder(annotation, exclude_unset=True)
key = annotation.key()
collection.document(key).set(annotation_json)
if move_key != "" and move_key != key:
collection.document(move_key).delete()
return key
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in put annotation for dataset {dataset}")
def delete_annotation(dataset: str,
key: str,
user_email: str = "",
user: User = Depends(get_user)):
""" Delete an annotation based on the key supplied in the URL. If the client has proper authorization,
annotations for another user can be deleted by supplying the query string '[email protected]'.
"""
if user_email == "":
user_email = user.email
authorized = (user_email == user.email and user.can_write_own(dataset)) or \
(user_email != user.email and user.can_write_others(dataset))
if not authorized:
raise HTTPException(
status_code=401,
detail=
f"no permission to delete annotation for user {user_email} on dataset {dataset}"
)
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_V2, dataset, user_email])
collection.document(key).delete()
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=
f"error in deleting annotation with key {key} for dataset {dataset}"
)
def delete_searches(dataset: str,
x: int,
y: int,
z: int,
user: User = Depends(get_user)):
if not user.has_role("clio_general", dataset):
raise HTTPException(
status_code=401,
detail=
f"user does not have permission to delete searches in dataset {dataset_id}"
)
try:
# delete only supported from interface
# (delete by dataset + user name + xyz)
collection = firestore.get_collection(
[CLIO_SAVEDSEARCHES, "USER", "searches"])
match_list = collection.where("email", "==", user.email).where(
"locationkey", "==", f"{x}_{y}_{z}").where("dataset", "==",
dataset).get()
for match in match_list:
match.reference.delete()
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in deleting saved searches for dataset {dataset}")
def get_kv(scope: str,
key: str,
timestamp: bool = False,
user: User = Depends(get_user)):
"""
Gets the key-value in the given scope with the given key for the authenticated user.
If timestamp = True, returns a _timestamp property with the Unix time of key-value persistence.
"""
try:
collection = firestore.get_collection(
[CLIO_KEYVALUE, user.email, scope])
value_ref = collection.document(key).get()
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving key-value for key {key}, scope {scope}"
)
if not value_ref.exists:
raise HTTPException(
status_code=404,
detail=f"no key-value found for key {key}, scope {scope}")
value = value_ref.to_dict()
if not timestamp:
del value["_timestamp"]
return value
def delete_atlas(dataset: str,
x: int,
y: int,
z: int,
user: User = Depends(get_user)):
if not user.can_write_own(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to delete annotations in dataset {dataset}")
try:
# delete only supported from interface
# (delete by dataset + user name + xyz)
collection = firestore.get_collection(
[CLIO_ANNOTATIONS, "ATLAS", "annotations"])
match_list = collection.where("user", "==", user.email).where(
"locationkey", "==", f"{x}_{y}_{z}").where("dataset", "==",
dataset).get()
for match in match_list:
match.reference.delete()
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=
f"error in deleting annotation ({x},{y},{z}) for dataset {dataset}"
)
def get_annotations(dataset: str,
groups: str = "",
user: str = "",
requestor: User = Depends(get_user)):
""" Returns all annotations for the user defined by the accompanying Authorization token
or the query string user if the requestor has admin permissions.
Return format is JSON list with annotations and generated fields "user" and "key",
where "key" is a user-scoped key used in DELETE requests for annotations.
Optional query string "groups" (names separated by commas) can result in larger set
of annotations returned, corresponding to all annotations for the given groups.
The groups can only be ones in which the user is a member unless user has admin permissions.
"""
if not requestor.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
if user == "":
user = requestor.email
elif user != requestor.email and not requestor.is_admin():
raise HTTPException(
status_code=401,
detail=f"get of user {user} requires admin permissions")
output = []
members = set([user])
if groups != "":
groups_queried = set(groups.split(','))
if len(groups_queried) > 0:
if requestor.is_admin():
groups_added = groups_queried
else:
groups_added = groups_queried.intersection(requestor.groups)
if len(groups_added) == 0:
raise HTTPException(
status_code=400,
detail=
f"requestor {requestor.email} is not member of requested groups {groups_queried}"
)
members.update(group_members(requestor, groups_added))
try:
for member in members:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_V2, dataset, member])
annotations_ref = collection.get()
for annotation_ref in annotations_ref:
annotation_dict = annotation_ref.to_dict()
annotation_dict["user"] = member
annotation_dict["key"] = annotation_ref.id
output.append(annotation_dict)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
return output
def _get_cache(collection_path: List[str], document: str) -> DocumentCache:
docpath = _pathname(collection_path, document)
if docpath in _caches:
doc_cache = _caches[docpath]
else:
collection = get_collection(collection_path)
ref = collection.document(document)
doc_cache = DocumentCache(ref, _max_stale_time)
_caches[docpath] = doc_cache
return doc_cache
def get_all_annotations(dataset: str,
annotation_type: str,
cursor: str = None,
size: int = MAX_ANNOTATIONS_RETURNED,
user: User = Depends(get_user)):
""" Returns all current neuron annotations for the given dataset and annotation type.
Query strings:
cursor (str): If supplied, annotations after the given id field are sent.
size (int): If supplied, at most this many annotations are returned.
Returns:
A JSON list of the annotations.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
if cursor:
cursor = f'id{cursor}' # convert id into key format
output = []
try:
t0 = time.time()
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_GLOBAL, annotation_type,
dataset]).where('_head', '==', True)
pagesize = min(size, 5000)
while True:
query = collection.limit(pagesize).order_by('__name__')
if cursor:
query = query.start_after({"__name__": cursor})
t1 = time.time()
retrieved = 0
for snapshot in query.stream():
retrieved += 1
annotation = remove_reserved_fields(snapshot.to_dict())
output.append(annotation)
cursor = snapshot.id
print(f'processed {retrieved} in {time.time() - t1} secs')
if retrieved < pagesize or len(output) == size:
break
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
print(f'{len(output)} total processed in {time.time() - t0} secs')
return output
def delete_kv(scope: str, key: str, user: User = Depends(get_user)):
"""Deletes a key-value with the given key in the given scope for the authenticated user."""
try:
collection = firestore.get_collection(
[CLIO_KEYVALUE, user.email, scope])
collection.document(key).delete()
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in deleting key-value for key {key}, scope {scope}")
def delete_users(deleted_emails: List, user: User = Depends(get_user)):
if not user.is_admin():
raise HTTPException(status_code=401,
detail="user lacks permission for /users endpoint")
try:
collection = firestore.get_collection([CLIO_USERS])
for email in deleted_emails:
collection.document(email).delete()
users.uncache_user(email)
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail=f"error in deleting users: {e}")
def pull_request(req: PullRequest, user: User = Depends(get_user)):
"""Copies user's pull request to protected area for later processing.
POSTed body should be JSON with pull string equal to path to kv with scope and key:
{"post": "kv/scope/key"}
"""
elems = req.pull.split('/')
if len(elems) != 3:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST,
detail=f"badly formatted pull request '{req.pull}'")
if elems[0] == 'kv':
scope = elems[1]
key = elems[2]
else:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST,
detail=f"only /kv pull requests supported at this time")
try:
collection = firestore.get_collection(
[CLIO_KEYVALUE, user.email, scope])
value_ref = collection.document(key).get()
if not value_ref.exists:
raise HTTPException(
status_code=HTTPStatus.NOT_FOUND,
detail=f"no pull request found corresponding to {req.pull}")
pulldata = value_ref.to_dict()
collection = firestore.get_collection(
[CLIO_PULL_REQUESTS, user.email, scope])
pulldata["_timestamp"] = time.time()
collection.document(key).set(pulldata)
except Exception as e:
print(e)
raise HTTPException(status_code=HTTPStatus.BAD_REQUEST,
detail=f"error attempting pull request: {e}")
def get_volume(volume: str, current_user: User = Depends(get_user)):
global cache
try:
if public_dataset(volume) or current_user.can_read(volume):
doc_ref = firestore.get_collection(CLIO_VOLUMES).document(
volume).get()
if doc_ref.exists:
data = doc_ref.to_dict()
cache[volume] = Volume(**data)
return data
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving volumes' metadata")
def pull_request(user_email: str, user: User = Depends(get_user)):
"""Returns user's pull requests. If "user_email" query string is provided, that user's pull
request is returned if the requester has sufficient permission. All pull requests can
be obtained by using the query string "user_email=all" if the user is admin.
{"kv/scope1/key1": value1, "kv/scope2/key2": value2}
"""
prs = {}
try:
collection = firestore.get_collection([CLIO_PULL_REQUESTS])
emails = set()
if user_email == "":
emails = set([user.email])
elif user_email == "all":
if not user.is_admin():
raise HTTPException(
status_code=401,
detail=f"no permission to access all pull requests")
emails = [ref.id for ref in collection.list_documents()]
else:
authorized = (user_email == user.email or user.is_admin())
if not authorized:
raise HTTPException(
status_code=401,
detail=
f"no permission to access pull requests for user {user_email}"
)
emails = set([user_email])
for email in emails:
user_ref = collection.document(email).get()
user_prs = {}
pr_types = user_ref.reference.collections()
for pr_type in pr_types:
user_prs[pr_type.id] = {}
for pr in pr_type.stream():
user_prs[pr_type.id][pr.id] = pr.to_dict()
if len(user_prs) != 0:
prs[email] = user_prs
except HTTPException as e:
raise e
except Exception as e:
print(e)
raise HTTPException(status_code=HTTPStatus.BAD_REQUEST,
detail=f"error attempting pull request: {e}")
return prs
def post_volumes(volumes: Dict[str, Volume],
current_user: User = Depends(get_user)):
global cache
if not current_user.is_admin():
raise HTTPException(status_code=401,
detail="user must be admin to set volume metadata")
try:
collection = firestore.get_collection(CLIO_VOLUMES)
for volume_id, volume in volumes.items():
data = volume.dict(exclude_unset=True)
collection.document(volume_id).set(data)
cache[volume_id] = Volume(**data)
except Exception as e:
print(e)
raise HTTPException(status_code=400, detail="error in POSTing volumes")
def get_volumes(current_user: User = Depends(get_user)):
global cache
try:
collection = firestore.get_collection(CLIO_VOLUMES)
volumes_out = {}
for volume in collection.stream():
volume_info = volume.to_dict()
cache[volume.id] = Volume(**volume_info)
if public_dataset(volume.id) or current_user.can_read(volume.id):
volumes_out[volume.id] = volume_info
return volumes_out
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving volumes' metadata")
def delete_datasets(to_delete: List[str],
current_user: User = Depends(get_user)):
if not current_user.is_admin():
raise HTTPException(
status_code=401,
detail="user must be admin to delete dataset metadata")
try:
collection = firestore.get_collection(CLIO_DATASETS)
for dataset_id in to_delete:
collection.document(dataset_id).delete()
# TODO -- Allow deletion of all data corresponding to this dataset?
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail=f"error in deleting datasets {to_delete}")
def get_annotations(dataset: str,
annotation_type: str,
id: str,
version: str = "",
changes: bool = False,
id_field: str = "bodyid",
user: User = Depends(get_user)):
""" Returns the neuron annotation associated with the given id.
Query strings:
version (str): If supplied, annotations are for the given dataset version.
changes (bool): If True, returns list of changes to this annotation across all versions.
id_field (str): The id field name (default: "bodyid")
Returns:
A JSON list (if changes requested or multiple ids given) or JSON object if not.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
if version != "":
cur_dataset = get_dataset(dataset)
if cur_dataset.tag == version:
version = ""
if "," in id:
id_strs = id.split(",")
ids = [int(id_str) for id_str in id_strs]
else:
ids = [int(id)]
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_GLOBAL, annotation_type, dataset])
return run_query_on_ids(collection, collection, ids, id_field, version,
changes)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
def post_kv(scope: str,
key: str,
payload: dict,
user: User = Depends(get_user)):
"""Puts a key-value in the given scope for the authenticated user."""
try:
payload["_timestamp"] = time.time()
collection = firestore.get_collection(
[CLIO_KEYVALUE, user.email, scope])
collection.document(key).set(payload)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error posting key-value for key {key}, scope {scope}: {e}"
)
def get_dataset(dataset: str,
templates: bool = False,
current_user: User = Depends(get_user)):
try:
if public_dataset(dataset) or current_user.can_read(dataset):
doc_ref = firestore.get_collection(CLIO_DATASETS).document(
dataset).get()
if not doc_ref.exists:
raise Exception(f'could not find dataset {dataset}')
if templates:
return doc_ref.to_dict()
else:
return replace_templates(doc_ref.to_dict())
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving datasets' metadata")
def post_users(postdata: Dict[str, UserPayload],
user: User = Depends(get_user)):
if not user.is_admin():
raise HTTPException(status_code=401,
detail="user lacks permission for /users endpoint")
try:
collection = firestore.get_collection([CLIO_USERS])
for email, data in postdata.items():
user_dict = data.dict()
collection.document(email).set(user_dict)
user_dict["email"] = email
user_with_email = User(**user_dict)
users.cache_user(user_with_email)
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail=f"error in posting users: {e}")
def submit_subvol(dataset: str, payload: dict, user: User = Depends(get_user)):
"""
Stores reference to an edited neuroglancer subvolume.
"""
if not user.can_write_own(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to submit subvolume in dataset {dataset}")
try:
payload["_timestamp"] = time.time()
collection = firestore.get_collection(
[CLIO_SUBVOL, user.email, "submissions"])
collection.document().set(payload)
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error submitting subvolume for dataset {dataset}: {e}")
def delete_volumes(to_delete: List[str],
current_user: User = Depends(get_user)):
global cache
if not current_user.is_admin():
raise HTTPException(
status_code=401,
detail="user must be admin to delete volume metadata")
try:
collection = firestore.get_collection(CLIO_VOLUMES)
for volume_id in to_delete:
collection.document(volume_id).delete()
if volume_id in cache:
del cache[volume_id]
# TODO -- Allow deletion of all data corresponding to this volume?
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail=f"error in deleting volumes {to_delete}")
def get_atlas(dataset: str, user: User = Depends(get_user)):
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS, "ATLAS", "annotations"])
if dataset != "all":
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations")
annotations = collection.where("user", "==", user.email).where(
"dataset", "==", dataset).get()
output = {}
for annotation in annotations:
res = annotation.to_dict()
res["id"] = annotation.id
output[res["locationkey"]] = res
return output
else:
annotations = collection.stream()
output = []
for annotation in annotations:
res = annotation.to_dict()
annot_dataset = res.get("dataset", "")
if user.can_read(annot_dataset):
if "verified" not in res:
print(f"bad atlas point, adding 'verified': {res}")
res["verified"] = False
annotation.reference.set(res)
res["id"] = annotation.id
if res["verified"] or res["user"] == user.email:
output.append(res)
elif user.can_write_others(annot_dataset):
output.append(res)
return output
except HTTPException as e:
raise e
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving atlas for dataset {dataset}: {e}")
return
def get_annotations(dataset: str,
groups: str = "",
user: User = Depends(get_user)):
""" Returns all annotations for the user defined by the accompanying Authorization token.
Return format is JSON object with annotations as enclosed key-value pairs. Keys are
used in move and delete operations.
Optional query string "groups" (names separated by commas) can result in larger set
of annotations returned, corresponding to all annotations for the given groups.
The groups can only be ones in which the user is a member.
"""
if not user.can_read(dataset):
raise HTTPException(
status_code=401,
detail=f"no permission to read annotations on dataset {dataset}")
output = {}
members = set([user.email])
if groups != "":
groups_queried = set(groups.split(','))
if len(groups_queried) > 0:
groups_added = groups_queried.intersection(user.groups)
if len(groups_added) == 0:
raise HTTPException(
status_code=400,
detail=
f"user {user.email} is not member of requested groups {groups_queried}"
)
members.update(group_members(user, groups_added))
try:
for member in members:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_V2, dataset, member])
annotations_ref = collection.get()
for annotation_ref in annotations_ref:
annotation_dict = annotation_ref.to_dict()
output[annotation_ref.id] = annotation_dict
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving annotations for dataset {dataset}: {e}"
)
return output
def post_annotations(dataset: str, annotation_type: str, payload: Union[List[Dict], Dict], id_field: str = "bodyid", \
replace: bool = False, conditional: str = "", version: str = "", user: User = Depends(get_user)):
""" Add either a single annotation object or a list of objects. All must be all in the
same dataset version.
Query strings:
replace (bool): If True (default False), posted values replace existing ones, so any non-existing
fields are removed.
id_field (str): The field name that corresponds to the id, e.g., "bodyid"
conditional (str): A field name or list of names separated by commas that should only be written
if the field is currently non-existant or empty.
version (str): The clio tag string corresponding to a version, e.g., "v0.3.1"
"""
try:
collection = firestore.get_collection(
[CLIO_ANNOTATIONS_GLOBAL, annotation_type, dataset])
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=
f"error in getting annotations collection for dataset {dataset}: {e}"
)
if isinstance(payload, dict):
payload = [payload]
check_reserved_fields(payload)
num = 0
conditional_fields = []
if bool(conditional):
conditional_fields = conditional.split(',')
for annotation in payload:
write_annotation(collection, annotation, replace, id_field,
conditional_fields, version, user)
num += 1
if num % 100 == 0:
print(
f"Wrote {num} {annotation_type} annotations to dataset {dataset}..."
)
def post_datasets(datasets: Dict[str, Dataset],
current_user: User = Depends(get_user)):
if not current_user.is_admin():
raise HTTPException(
status_code=401,
detail="user must be admin to set dataset metadata")
try:
collection = firestore.get_collection(CLIO_DATASETS)
for dataset_id, dataset in datasets.items():
cached_dataset = get_cached_dataset(dataset_id)
if cached_dataset and cached_dataset.tag and cached_dataset.tag > dataset.tag:
raise Exception(
f'posted dataset {dataset_id} has tag {dataset.tag} earlier than current {cached_dataset.tag}'
)
collection.document(dataset_id).set(
dataset.dict(exclude_unset=True))
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in POSTing datasets: {e}")
def get_datasets(templates: bool = False,
current_user: User = Depends(get_user)):
try:
collection = firestore.get_collection(CLIO_DATASETS)
t0 = time.time()
datasets_out = {}
for dataset in collection.stream():
dataset_info = dataset.to_dict()
if public_dataset(dataset.id) or current_user.can_read(dataset.id):
if templates:
datasets_out[dataset.id] = dataset_info
else:
datasets_out[dataset.id] = replace_templates(dataset_info)
print(
f'Retrieved {len(datasets_out)} datasets in {time.time() - t0} seconds'
)
return datasets_out
except Exception as e:
print(e)
raise HTTPException(status_code=400,
detail="error in retrieving datasets' metadata")
def get_searches(dataset: str, user: User = Depends(get_user)):
if not user.has_role("clio_general", dataset):
raise HTTPException(
status_code=401,
detail=f"user does not have permission to read dataset {dataset_id}"
)
try:
collection = firestore.get_collection(
[CLIO_SAVEDSEARCHES, "USER", "searches"])
searches = collection.where("email", "==",
user.email).where("dataset", "==",
dataset).get()
output = {}
for search in searches:
res = search.to_dict()
res["id"] = search.id
output[res["locationkey"]] = res
return output
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in getting saved searches for dataset {dataset}")
def get_kv_all(scope: str,
timestamp: bool = False,
user: User = Depends(get_user)) -> dict:
"""
Gets all key-values in the given scope for the authenticated user. If timestamp = True, returns
a _timestamp property with the Unix time of key-value persistence.
"""
try:
collection = firestore.get_collection(
[CLIO_KEYVALUE, user.email, scope])
kvs = collection.get()
kvs_out = {}
for kv_ref in kvs:
value = kv_ref.to_dict()
if not timestamp:
del value["_timestamp"]
kvs_out[kv_ref.id] = value
except Exception as e:
print(e)
raise HTTPException(
status_code=400,
detail=f"error in retrieving key-values for scope {scope}")
return kvs_out
