def handle_root(table_id, atomic_id):
current_app.table_id = table_id
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
# Convert seconds since epoch to UTC datetime
try:
timestamp = float(request.args.get("timestamp", time.time()))
timestamp = datetime.fromtimestamp(timestamp, UTC)
except (TypeError, ValueError) as e:
raise (cg_exceptions.BadRequest(
"Timestamp parameter is not a valid unix timestamp"))
stop_layer = request.args.get("stop_layer", None)
if stop_layer is not None:
try:
stop_layer = int(stop_layer)
except (TypeError, ValueError) as e:
raise (cg_exceptions.BadRequest("stop_layer is not an integer"))
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
root_id = cg.get_root(np.uint64(atomic_id),
stop_layer=stop_layer,
time_stamp=timestamp)
# Return root ID
return root_id
def handle_split_preview(table_id):
current_app.table_id = table_id
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
data = json.loads(request.data)
current_app.logger.debug(data)
cg = app_utils.get_cg(table_id)
data_dict = {}
for k in ["sources", "sinks"]:
data_dict[k] = collections.defaultdict(list)
for node in data[k]:
node_id = node[0]
x, y, z = node[1:]
coordinate = np.array([x, y, z]) / cg.segmentation_resolution
atomic_id = cg.get_atomic_id_from_coord(
coordinate[0],
coordinate[1],
coordinate[2],
parent_id=np.uint64(node_id))
if atomic_id is None:
raise cg_exceptions.BadRequest(
f"Could not determine supervoxel ID for coordinates "
f"{coordinate}.")
data_dict[k]["id"].append(atomic_id)
data_dict[k]["coord"].append(coordinate)
current_app.logger.debug(data_dict)
try:
supervoxel_ccs, illegal_split = cg._run_multicut(
source_ids=data_dict["sources"]["id"],
sink_ids=data_dict["sinks"]["id"],
source_coords=data_dict["sources"]["coord"],
sink_coords=data_dict["sinks"]["coord"],
bb_offset=(240, 240, 24),
split_preview=True)
except cg_exceptions.PreconditionError as e:
raise cg_exceptions.BadRequest(str(e))
resp = {
"supervoxel_connected_components": supervoxel_ccs,
"illegal_split": illegal_split
}
return resp
def handle_redo(table_id):
if table_id in ["fly_v26", "fly_v31"]:
raise cg_exceptions.InternalServerError(
"Redo not supported for this chunkedgraph table.")
current_app.table_id = table_id
data = json.loads(request.data)
is_priority = request.args.get('priority', True, type=str2bool)
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
current_app.logger.debug(data)
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
operation_id = np.uint64(data["operation_id"])
try:
ret = cg.redo(user_id=user_id, operation_id=operation_id)
except cg_exceptions.LockingError as e:
raise cg_exceptions.InternalServerError(
"Could not acquire root lock for redo operation.")
except (cg_exceptions.PreconditionError,
cg_exceptions.PostconditionError) as e:
raise cg_exceptions.BadRequest(str(e))
current_app.logger.debug(("after redo:", ret.new_root_ids))
current_app.logger.debug(("lvl2_nodes:", ret.new_lvl2_ids))
if ret.new_lvl2_ids.size > 0:
trigger_remesh(table_id, ret.new_lvl2_ids, is_priority=is_priority)
return ret
def handle_rollback(table_id):
if table_id in ["fly_v26", "fly_v31"]:
raise cg_exceptions.InternalServerError(
"Rollback not supported for this chunkedgraph table.")
current_app.table_id = table_id
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
target_user_id = request.args["user_id"]
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
user_operations = all_user_operations(table_id)
operation_ids = user_operations["operation_id"]
timestamps = user_operations["timestamp"]
operations = list(zip(operation_ids, timestamps))
operations.sort(key=lambda op: op[1])
for operation in operations:
operation_id = operation[0]
try:
ret = cg.undo_operation(user_id=target_user_id,
operation_id=operation_id)
except cg_exceptions.LockingError as e:
raise cg_exceptions.InternalServerError(
"Could not acquire root lock for undo operation.")
except (cg_exceptions.PreconditionError,
cg_exceptions.PostconditionError) as e:
raise cg_exceptions.BadRequest(str(e))
if ret.new_lvl2_ids.size > 0:
trigger_remesh(table_id, ret.new_lvl2_ids, is_priority=False)
return user_operations
def all_user_operations(table_id):
current_app.table_id = table_id
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
target_user_id = request.args["user_id"]
try:
start_time = float(request.args.get("start_time", 0))
start_time = datetime.fromtimestamp(start_time, UTC)
except (TypeError, ValueError):
raise (cg_exceptions.BadRequest(
"start_time parameter is not a valid unix timestamp"))
# Call ChunkedGraph
cg_instance = app_utils.get_cg(table_id)
log_rows = cg_instance.read_log_rows(start_time=start_time)
valid_entry_ids = []
timestamp_list = []
entry_ids = np.sort(list(log_rows.keys()))
for entry_id in entry_ids:
entry = log_rows[entry_id]
user_id = entry[column_keys.OperationLogs.UserID]
if user_id == target_user_id:
valid_entry_ids.append(entry_id)
timestamp = entry["timestamp"]
timestamp_list.append(timestamp)
return {"operation_id": valid_entry_ids, "timestamp": timestamp_list}
def tabular_change_log_recent(table_id):
current_app.table_id = table_id
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
try:
start_time = float(request.args.get("start_time", 0))
start_time = datetime.fromtimestamp(start_time, UTC)
except (TypeError, ValueError):
raise (cg_exceptions.BadRequest(
"start_time parameter is not a valid unix timestamp"))
# Call ChunkedGraph
cg_instance = app_utils.get_cg(table_id)
log_rows = cg_instance.read_log_rows(start_time=start_time)
timestamp_list = []
user_list = []
entry_ids = np.sort(list(log_rows.keys()))
for entry_id in entry_ids:
entry = log_rows[entry_id]
timestamp = entry["timestamp"]
timestamp_list.append(timestamp)
user_id = entry[column_keys.OperationLogs.UserID]
user_list.append(user_id)
return pd.DataFrame.from_dict({
"operation_id": entry_ids,
"timestamp": timestamp_list,
"user_id": user_list
})
def handle_roots(table_id, is_binary=False):
current_app.request_type = "roots"
current_app.table_id = table_id
if is_binary:
node_ids = np.frombuffer(request.data, np.uint64)
else:
node_ids = np.array(json.loads(request.data)["node_ids"],
dtype=np.uint64)
# Convert seconds since epoch to UTC datetime
try:
timestamp = float(request.args.get("timestamp", time.time()))
timestamp = datetime.fromtimestamp(timestamp, UTC)
except (TypeError, ValueError) as e:
raise (cg_exceptions.BadRequest("Timestamp parameter is not a valid"
" unix timestamp"))
stop_layer = request.args.get("stop_layer", None)
if stop_layer is not None:
stop_layer = int(stop_layer)
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
root_ids = cg.get_roots(node_ids,
stop_layer=stop_layer,
time_stamp=timestamp)
return root_ids
def handle_root_2(table_id, atomic_id):
# Convert seconds since epoch to UTC datetime
try:
timestamp = float(request.args.get('timestamp', time.time()))
timestamp = datetime.fromtimestamp(timestamp, UTC)
except (TypeError, ValueError) as e:
raise(cg_exceptions.BadRequest("Timestamp parameter is not a valid"
" unix timestamp"))
return handle_root_main(table_id, np.uint64(atomic_id), timestamp)
def _get_supervoxel_id_from_node(node):
node_id = node[0]
x, y, z = node[1:]
coordinate = np.array([x, y, z]) / cg.segmentation_resolution
supervoxel_id = cg.get_atomic_id_from_coord(
coordinate[0],
coordinate[1],
coordinate[2],
parent_id=np.uint64(node_id))
if supervoxel_id is None:
raise cg_exceptions.BadRequest(
f"Could not determine supervoxel ID for coordinates "
f"{coordinate}.")
return supervoxel_id
def merge_log(table_id, root_id):
current_app.table_id = table_id
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
try:
time_stamp_past = float(request.args.get("timestamp", 0))
time_stamp_past = datetime.fromtimestamp(time_stamp_past, UTC)
except (TypeError, ValueError) as e:
raise (cg_exceptions.BadRequest("Timestamp parameter is not a valid"
" unix timestamp"))
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
segment_history = cg_history.SegmentHistory(cg, int(root_id))
return segment_history.merge_log(correct_for_wrong_coord_type=True)
def change_log(table_id, root_id):
current_app.request_type = "change_log"
try:
time_stamp_past = float(request.args.get('timestamp', 0))
time_stamp_past = datetime.fromtimestamp(time_stamp_past, UTC)
except (TypeError, ValueError) as e:
raise(cg_exceptions.BadRequest("Timestamp parameter is not a valid"
" unix timestamp"))
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
change_log = cg.get_change_log(root_id=np.uint64(root_id),
correct_for_wrong_coord_type=True,
time_stamp_past=time_stamp_past)
return jsonify(change_log)
def change_log(table_id, root_id=None):
current_app.table_id = table_id
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
try:
time_stamp_past = float(request.args.get("timestamp", 0))
time_stamp_past = datetime.fromtimestamp(time_stamp_past, UTC)
except (TypeError, ValueError) as e:
raise (cg_exceptions.BadRequest("Timestamp parameter is not a valid"
" unix timestamp"))
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
if not root_id:
return cg_history.get_all_log_entries(cg)
segment_history = cg_history.SegmentHistory(cg, int(root_id))
return segment_history.change_log()
def handle_l2_chunk_children(table_id, chunk_id, as_array):
current_app.request_type = "l2_chunk_children"
current_app.table_id = table_id
# Convert seconds since epoch to UTC datetime
try:
timestamp = float(request.args.get("timestamp", time.time()))
timestamp = datetime.fromtimestamp(timestamp, UTC)
except (TypeError, ValueError) as e:
raise (cg_exceptions.BadRequest("Timestamp parameter is not a valid"
" unix timestamp"))
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
chunk_layer = cg.get_chunk_layer(chunk_id)
if chunk_layer != 2:
raise (cg_exceptions.PreconditionError(
f'This function only accepts level 2 chunks, the chunk requested is a level {chunk_layer} chunk'
))
rr_chunk = cg.range_read_chunk(chunk_id=np.uint64(chunk_id),
columns=column_keys.Hierarchy.Child,
time_stamp=timestamp)
if as_array:
l2_chunk_array = []
for l2 in rr_chunk:
svs = rr_chunk[l2][0].value
for sv in svs:
l2_chunk_array.extend([l2, sv])
return np.array(l2_chunk_array)
else:
# store in dict of keys to arrays to remove reliance on bigtable
l2_chunk_dict = {}
for k in rr_chunk:
l2_chunk_dict[k] = rr_chunk[k][0].value
return l2_chunk_dict
def handle_pairwise_contact_sites(table_id, first_node_id, second_node_id):
current_app.request_type = "pairwise_contact_sites"
current_app.table_id = table_id
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
try:
timestamp = float(request.args.get("timestamp", time.time()))
timestamp = datetime.fromtimestamp(timestamp, UTC)
except (TypeError, ValueError) as e:
raise (cg_exceptions.BadRequest("Timestamp parameter is not a valid"
" unix timestamp"))
exact_location = request.args.get("exact_location",
True,
type=app_utils.toboolean)
cg = app_utils.get_cg(table_id)
contact_sites_list, cs_metadata = contact_sites.get_contact_sites_pairwise(
cg,
np.uint64(first_node_id),
np.uint64(second_node_id),
end_time=timestamp,
exact_location=exact_location,
)
return contact_sites_list, cs_metadata
def handle_contact_sites(table_id, root_id):
partners = request.args.get("partners", True, type=app_utils.toboolean)
as_list = request.args.get("as_list", True, type=app_utils.toboolean)
areas_only = request.args.get("areas_only", True, type=app_utils.toboolean)
current_app.table_id = table_id
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
try:
timestamp = float(request.args.get("timestamp", time.time()))
timestamp = datetime.fromtimestamp(timestamp, UTC)
except (TypeError, ValueError) as e:
raise (cg_exceptions.BadRequest("Timestamp parameter is not a valid"
" unix timestamp"))
if "bounds" in request.args:
bounds = request.args["bounds"]
bounding_box = np.array([b.split("-") for b in bounds.split("_")],
dtype=np.int).T
else:
bounding_box = None
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
cs_list, cs_metadata = contact_sites.get_contact_sites(
cg,
np.uint64(root_id),
bounding_box=bounding_box,
compute_partner=partners,
end_time=timestamp,
as_list=as_list,
areas_only=areas_only)
return cs_list, cs_metadata
def handle_is_latest_roots(table_id, is_binary):
current_app.request_type = "is_latest_roots"
current_app.table_id = table_id
if is_binary:
node_ids = np.frombuffer(request.data, np.uint64)
else:
node_ids = np.array(json.loads(request.data)["node_ids"],
dtype=np.uint64)
# Convert seconds since epoch to UTC datetime
try:
timestamp = float(request.args.get("timestamp", time.time()))
timestamp = datetime.fromtimestamp(timestamp, UTC)
except (TypeError, ValueError) as e:
raise (cg_exceptions.BadRequest("Timestamp parameter is not a valid"
" unix timestamp"))
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
row_dict = cg.read_node_id_rows(node_ids=node_ids,
columns=column_keys.Hierarchy.NewParent)
is_latest = ~np.isin(node_ids, list(row_dict.keys()))
return is_latest
def handle_split(table_id):
current_app.table_id = table_id
data = json.loads(request.data)
is_priority = request.args.get('priority', True, type=str2bool)
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
current_app.logger.debug(data)
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
data_dict = {}
for k in ["sources", "sinks"]:
data_dict[k] = collections.defaultdict(list)
for node in data[k]:
node_id = node[0]
x, y, z = node[1:]
coordinate = np.array([x, y, z]) / cg.segmentation_resolution
atomic_id = cg.get_atomic_id_from_coord(
coordinate[0],
coordinate[1],
coordinate[2],
parent_id=np.uint64(node_id),
)
if atomic_id is None:
raise cg_exceptions.BadRequest(
f"Could not determine supervoxel ID for coordinates "
f"{coordinate}.")
data_dict[k]["id"].append(atomic_id)
data_dict[k]["coord"].append(coordinate)
current_app.logger.debug(data_dict)
try:
ret = cg.remove_edges(
user_id=user_id,
source_ids=data_dict["sources"]["id"],
sink_ids=data_dict["sinks"]["id"],
source_coords=data_dict["sources"]["coord"],
sink_coords=data_dict["sinks"]["coord"],
mincut=True,
)
except cg_exceptions.LockingError as e:
raise cg_exceptions.InternalServerError(
"Could not acquire root lock for split operation.")
except cg_exceptions.PreconditionError as e:
raise cg_exceptions.BadRequest(str(e))
if ret.new_root_ids is None:
raise cg_exceptions.InternalServerError(
"Could not split selected segment groups.")
current_app.logger.debug(("after split:", ret.new_root_ids))
current_app.logger.debug(("lvl2_nodes:", ret.new_lvl2_ids))
if len(ret.new_lvl2_ids) > 0:
trigger_remesh(table_id, ret.new_lvl2_ids, is_priority=is_priority)
return ret
def handle_split(table_id):
current_app.request_type = "split"
data = json.loads(request.data)
user_id = str(request.remote_addr)
current_app.logger.debug(data)
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
data_dict = {}
for k in ["sources", "sinks"]:
data_dict[k] = collections.defaultdict(list)
for node in data[k]:
node_id = node[0]
x, y, z = node[1:]
x /= 2
y /= 2
coordinate = np.array([x, y, z])
current_app.logger.debug(("before", coordinate))
if not cg.is_in_bounds(coordinate):
coordinate /= cg.segmentation_resolution
coordinate[0] *= 2
coordinate[1] *= 2
current_app.logger.debug(("after", coordinate))
atomic_id = cg.get_atomic_id_from_coord(coordinate[0],
coordinate[1],
coordinate[2],
parent_id=np.uint64(
node_id))
if atomic_id is None:
raise cg_exceptions.BadRequest(
f"Could not determine supervoxel ID for coordinates "
f"{coordinate}.")
data_dict[k]["id"].append(atomic_id)
data_dict[k]["coord"].append(coordinate)
current_app.logger.debug(data_dict)
lvl2_nodes = []
try:
ret = cg.remove_edges(user_id=user_id,
source_ids=data_dict["sources"]["id"],
sink_ids=data_dict["sinks"]["id"],
source_coords=data_dict["sources"]["coord"],
sink_coords=data_dict["sinks"]["coord"],
mincut=True,
return_new_lvl2_nodes=True,
remesh_preview=False)
if len(ret) == 2:
new_roots, lvl2_nodes = ret
else:
new_roots = ret
except cg_exceptions.LockingError as e:
raise cg_exceptions.InternalServerError(
"Could not acquire root lock for split operation.")
except cg_exceptions.PreconditionError as e:
raise cg_exceptions.BadRequest(str(e))
if new_roots is None:
raise cg_exceptions.InternalServerError(
"Could not split selected segment groups."
)
current_app.logger.debug(("after split:", new_roots))
t = threading.Thread(target=meshing_app_blueprint._mesh_lvl2_nodes,
args=(cg.get_serialized_info(), lvl2_nodes))
t.start()
# Return binary
return app_utils.tobinary(new_roots)
def handle_merge(table_id):
current_app.request_type = "merge"
nodes = json.loads(request.data)
user_id = str(request.remote_addr)
current_app.logger.debug(nodes)
assert len(nodes) == 2
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
atomic_edge = []
coords = []
for node in nodes:
node_id = node[0]
x, y, z = node[1:]
x /= 2
y /= 2
coordinate = np.array([x, y, z])
if not cg.is_in_bounds(coordinate):
coordinate /= cg.segmentation_resolution
coordinate[0] *= 2
coordinate[1] *= 2
atomic_id = cg.get_atomic_id_from_coord(coordinate[0],
coordinate[1],
coordinate[2],
parent_id=np.uint64(node_id))
if atomic_id is None:
raise cg_exceptions.BadRequest(
f"Could not determine supervoxel ID for coordinates "
f"{coordinate}."
)
coords.append(coordinate)
atomic_edge.append(atomic_id)
# Protection from long range mergers
chunk_coord_delta = cg.get_chunk_coordinates(atomic_edge[0]) - \
cg.get_chunk_coordinates(atomic_edge[1])
if np.any(np.abs(chunk_coord_delta) > 3):
raise cg_exceptions.BadRequest(
"Chebyshev distance between merge points exceeded allowed maximum "
"(3 chunks).")
lvl2_nodes = []
try:
ret = cg.add_edges(user_id=user_id,
atomic_edges=np.array(atomic_edge,
dtype=np.uint64),
source_coord=coords[:1],
sink_coord=coords[1:],
return_new_lvl2_nodes=True,
remesh_preview=False)
if len(ret) == 2:
new_root, lvl2_nodes = ret
else:
new_root = ret
except cg_exceptions.LockingError as e:
raise cg_exceptions.InternalServerError(
"Could not acquire root lock for merge operation.")
except cg_exceptions.PreconditionError as e:
raise cg_exceptions.BadRequest(str(e))
if new_root is None:
raise cg_exceptions.InternalServerError(
"Could not merge selected supervoxel.")
t = threading.Thread(target=meshing_app_blueprint._mesh_lvl2_nodes,
args=(cg.get_serialized_info(), lvl2_nodes))
t.start()
# Return binary
return app_utils.tobinary(new_root)
def handle_merge(table_id):
current_app.table_id = table_id
nodes = json.loads(request.data)
is_priority = request.args.get('priority', True, type=str2bool)
user_id = str(g.auth_user["id"])
current_app.user_id = user_id
current_app.logger.debug(nodes)
assert len(nodes) == 2
# Call ChunkedGraph
cg = app_utils.get_cg(table_id)
atomic_edge = []
coords = []
for node in nodes:
node_id = node[0]
x, y, z = node[1:]
coordinate = np.array([x, y, z]) / cg.segmentation_resolution
atomic_id = cg.get_atomic_id_from_coord(coordinate[0],
coordinate[1],
coordinate[2],
parent_id=np.uint64(node_id))
if atomic_id is None:
raise cg_exceptions.BadRequest(
f"Could not determine supervoxel ID for coordinates "
f"{coordinate}.")
coords.append(coordinate)
atomic_edge.append(atomic_id)
# Protection from long range mergers
chunk_coord_delta = cg.get_chunk_coordinates(
atomic_edge[0]) - cg.get_chunk_coordinates(atomic_edge[1])
if np.any(np.abs(chunk_coord_delta) > 3):
raise cg_exceptions.BadRequest(
"Chebyshev distance between merge points exceeded allowed maximum "
"(3 chunks).")
try:
ret = cg.add_edges(
user_id=user_id,
atomic_edges=np.array(atomic_edge, dtype=np.uint64),
source_coord=coords[:1],
sink_coord=coords[1:],
)
except cg_exceptions.LockingError as e:
raise cg_exceptions.InternalServerError(
"Could not acquire root lock for merge operation.")
except cg_exceptions.PreconditionError as e:
raise cg_exceptions.BadRequest(str(e))
if ret.new_root_ids is None:
raise cg_exceptions.InternalServerError("Could not merge selected "
"supervoxel.")
current_app.logger.debug(("lvl2_nodes:", ret.new_lvl2_ids))
if len(ret.new_lvl2_ids) > 0:
trigger_remesh(table_id, ret.new_lvl2_ids, is_priority=is_priority)
return ret
