def run_export(self):
self.progressSignal(0)
url = self.NodeDataUrl.value
url_path = url.split('://')[1]
hostname, api, node, uuid, dataname = url_path.split('/')
assert api == 'api'
assert node == 'node'
axiskeys = self.Input.meta.getAxisKeys()
shape = self.Input.meta.shape
if self._transpose_axes:
axiskeys = reversed(axiskeys)
shape = tuple(reversed(shape))
axiskeys = "".join( axiskeys )
if self.OffsetCoord.ready():
offset_start = self.OffsetCoord.value
else:
offset_start = (0,) * len( self.Input.meta.shape )
self.progressSignal(5)
# Get the dataset details
try:
metadata = VoxelsAccessor.get_metadata(hostname, uuid, dataname)
except VoxelsAccessor.BadRequestError as ex:
# Dataset doesn't exist yet. Let's create it.
metadata = VoxelsMetadata.create_default_metadata( shape,
self.Input.meta.dtype,
axiskeys,
0.0,
"" )
VoxelsAccessor.create_new(hostname, uuid, dataname, metadata)
# Since this class is generally used to push large blocks of data,
# we'll be nice and set throttle=True
client = VoxelsAccessor( hostname, uuid, dataname, throttle=True )
def handle_block_result(roi, data):
# Send it to dvid
roi = numpy.asarray(roi)
roi += offset_start
start, stop = roi
if self._transpose_axes:
data = data.transpose()
start = tuple(reversed(start))
stop = tuple(reversed(stop))
client.post_ndarray( start, stop, data )
requester = BigRequestStreamer( self.Input, roiFromShape( self.Input.meta.shape ) )
requester.resultSignal.subscribe( handle_block_result )
requester.progressSignal.subscribe( self.progressSignal )
requester.execute()
self.progressSignal(100)
def run_export(self):
self.progressSignal(0)
url = self.NodeDataUrl.value
url_path = url.split('://')[1]
hostname, api, node, uuid, dataname = url_path.split('/')
assert api == 'api'
assert node == 'node'
axiskeys = self.Input.meta.getAxisKeys()
shape = self.Input.meta.shape
if self._transpose_axes:
axiskeys = reversed(axiskeys)
shape = tuple(reversed(shape))
axiskeys = "".join(axiskeys)
if self.OffsetCoord.ready():
offset_start = self.OffsetCoord.value
else:
offset_start = (0, ) * len(self.Input.meta.shape)
self.progressSignal(5)
# Get the dataset details
try:
metadata = VoxelsAccessor.get_metadata(hostname, uuid, dataname)
except DVIDException as ex:
if ex.status != 404:
raise
# Dataset doesn't exist yet. Let's create it.
metadata = VoxelsMetadata.create_default_metadata(
shape, self.Input.meta.dtype, axiskeys, 0.0, "")
VoxelsAccessor.create_new(hostname, uuid, dataname, metadata)
# Since this class is generally used to push large blocks of data,
# we'll be nice and set throttle=True
client = VoxelsAccessor(hostname, uuid, dataname, throttle=True)
def handle_block_result(roi, data):
# Send it to dvid
roi = numpy.asarray(roi)
roi += offset_start
start, stop = roi
if self._transpose_axes:
data = data.transpose()
start = tuple(reversed(start))
stop = tuple(reversed(stop))
client.post_ndarray(start, stop, data)
requester = BigRequestStreamer(self.Input,
roiFromShape(self.Input.meta.shape))
requester.resultSignal.subscribe(handle_block_result)
requester.progressSignal.subscribe(self.progressSignal)
requester.execute()
self.progressSignal(100)
def _update_subvol_widget(self, node_uuid, dataname, typename):
"""
Update the subvolume widget with the min/max extents of the given node and dataname.
Note: The node and dataname do not necessarily have to match the currently
selected node and dataname.
This enables the right-click behavior, which can be used to
limit your data volume to the size of a different data volume.
"""
error_msg = None
try:
if typename == "roi":
node_service = DVIDNodeService(self._hostname, str(node_uuid))
roi_blocks_zyx = numpy.array(
node_service.get_roi(str(dataname)))
maxindex = tuple(DVID_BLOCK_WIDTH *
(1 + numpy.max(roi_blocks_zyx, axis=0)))
minindex = (0, 0, 0) # Rois are always 3D
axiskeys = "zyx"
# If the current selection is a dataset, then include a channel dimension
if self.get_selection().typename != "roi":
axiskeys = "zyxc"
minindex = minindex + (0, )
maxindex = maxindex + (
1,
) # FIXME: This assumes that the selected data has only 1 channel...
else:
# Query the server
raw_metadata = VoxelsAccessor.get_metadata(
self._hostname, node_uuid, dataname)
voxels_metadata = VoxelsMetadata(raw_metadata)
maxindex = voxels_metadata.shape
minindex = voxels_metadata.minindex
axiskeys = voxels_metadata.axiskeys
# If the current selection is a roi, then remove the channel dimension
if self.get_selection().typename == "roi":
axiskeys = "zyx"
minindex = minindex[:-1]
maxindex = maxindex[:-1]
except (DVIDException, ErrMsg) as ex:
error_msg = str(ex)
log_exception(logger)
else:
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True)
if error_msg:
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False)
QMessageBox.critical(self, "DVID Error", error_msg)
self._subvol_widget.initWithExtents("", (), (), ())
return
self._subvol_widget.initWithExtents(axiskeys, maxindex, minindex,
maxindex)
def _update_subvol_widget(self, node_uuid, dataname, typename):
"""
Update the subvolume widget with the min/max extents of the given node and dataname.
Note: The node and dataname do not necessarily have to match the currently
selected node and dataname.
This enables the right-click behavior, which can be used to
limit your data volume to the size of a different data volume.
"""
error_msg = None
try:
if typename == "roi":
node_service = DVIDNodeService(self._hostname, str(node_uuid))
roi_blocks_xyz = numpy.array(node_service.get_roi(str(dataname)))
maxindex = tuple(DVID_BLOCK_WIDTH * (1 + numpy.max(roi_blocks_xyz, axis=0)))
minindex = (0, 0, 0) # Rois are always 3D
axiskeys = "xyz"
# If the current selection is a dataset, then include a channel dimension
if self.get_selection().typename != "roi":
axiskeys = "cxyz"
minindex = (0,) + minindex
maxindex = (1,) + maxindex # FIXME: This assumes that the selected data has only 1 channel...
else:
# Query the server
raw_metadata = VoxelsAccessor.get_metadata(self._hostname, node_uuid, dataname)
voxels_metadata = VoxelsMetadata(raw_metadata)
maxindex = voxels_metadata.shape
minindex = voxels_metadata.minindex
axiskeys = voxels_metadata.axiskeys
# If the current selection is a roi, then remove the channel dimension
if self.get_selection().typename == "roi":
axiskeys = "xyz"
minindex = minindex[1:]
maxindex = maxindex[1:]
except (DVIDException, ErrMsg) as ex:
error_msg = str(ex)
log_exception(logger)
else:
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True)
if error_msg:
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False)
QMessageBox.critical(self, "DVID Error", error_msg)
self._subvol_widget.initWithExtents("", (), (), ())
return
self._subvol_widget.initWithExtents(axiskeys, maxindex, minindex, maxindex)
def _update_display(self):
super(DvidDataSelectionBrowser, self)._update_display()
hostname, dset_uuid, dataname, node_uuid = self.get_selection()
enable_contents = self._repos_info is not None and dataname != "" and node_uuid != ""
self._roi_groupbox.setEnabled(enable_contents)
if not dataname or not node_uuid:
self._roi_widget.initWithExtents("", (), (), ())
return
error_msg = None
try:
# Query the server
raw_metadata = VoxelsAccessor.get_metadata(hostname, node_uuid,
dataname)
voxels_metadata = VoxelsMetadata(raw_metadata)
except DVIDException as ex:
error_msg = ex.message
except ErrMsg as ex:
error_msg = str(ErrMsg)
except VoxelsAccessor.BadRequestError as ex:
# DVID will return an error if the selected dataset
# isn't a 'voxels' dataset and thus has no voxels metadata
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False)
return
else:
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True)
if error_msg:
QMessageBox.critical(self, "DVID Error", error_msg)
self._roi_widget.initWithExtents("", (), (), ())
return
self._roi_widget.initWithExtents(voxels_metadata.axiskeys,
voxels_metadata.shape,
voxels_metadata.minindex,
voxels_metadata.shape)
def _update_display(self):
super(DvidDataSelectionBrowser, self)._update_display()
hostname, dset_uuid, dataname, node_uuid = self.get_selection()
enable_contents = self._repos_info is not None and dataname != "" and node_uuid != ""
self._roi_groupbox.setEnabled(enable_contents)
if not dataname or not node_uuid:
self._roi_widget.initWithExtents("", (), (), ())
return
error_msg = None
try:
# Query the server
raw_metadata = VoxelsAccessor.get_metadata(hostname, node_uuid, dataname)
voxels_metadata = VoxelsMetadata(raw_metadata)
except DVIDException as ex:
error_msg = ex.message
except ErrMsg as ex:
error_msg = str(ErrMsg)
except VoxelsAccessor.BadRequestError as ex:
# DVID will return an error if the selected dataset
# isn't a 'voxels' dataset and thus has no voxels metadata
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False)
return
else:
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True)
if error_msg:
QMessageBox.critical(self, "DVID Error", error_msg)
self._roi_widget.initWithExtents("", (), (), ())
return
self._roi_widget.initWithExtents(
voxels_metadata.axiskeys, voxels_metadata.shape, voxels_metadata.minindex, voxels_metadata.shape
)
def main():
# Read cmd-line args
parser = argparse.ArgumentParser()
parser.add_argument("--hostname", default="localhost:8000")
parser.add_argument("--uuid", required=False, help="The node to upload to. If not provided, a new repo will be created (see --new-repo-alias).")
parser.add_argument("--data-name", required=False, help="The name of the data instance to modify. If it doesn't exist, it will be created first.")
parser.add_argument("--new-repo-alias", required=False, help="If no uuid is provided, a new repo is created, with this name.")
parser.add_argument("input_file", help="For example: /tmp/myfile.h5/dataset")
args = parser.parse_args()
if '.h5' not in args.input_file:
sys.stderr.write("File name does not indicate hdf5.\n")
sys.exit(1)
filepath, dset_name = args.input_file.split('.h5')
filepath += '.h5'
if not dset_name:
sys.stderr.write("You must provide a dataset name, e.g. myfile.h5/mydataset\n")
sys.exit(1)
if not os.path.exists(filepath):
sys.stderr.write("File doesn't exist: {}\n".format(filepath))
sys.exit(1)
# If no uuid given, create a new repo on the server
uuid = args.uuid
if uuid is None:
alias = args.new_repo_alias or "testrepo"
server = DVIDServerService(args.hostname)
uuid = server.create_new_repo(alias, "This is a test repo loaded with data from ".format(args.input_file))
uuid = str(uuid)
# Read the input data from the file
print("Reading {}{}".format( filepath, dset_name ))
with h5py.File(filepath) as f_in:
data = f_in[dset_name][:]
# We assume data is 3D or 4D, in C-order
# We adjust it to 4D, fortran-order
if data.ndim == 3:
data = data[...,None]
data = data.transpose()
assert data.flags['F_CONTIGUOUS'], "Data is not contiguous!"
assert data.ndim == 4, "Data must be 3D with axes zyx or 4D with axes zyxc (C-order)"
assert data.shape[0] == 1, "Data must have exactly 1 channel, not {}".format( data.shape[0] )
# Choose a default data instance name if necessary
if data.dtype == numpy.uint8:
data_name = args.data_name or "grayscale"
elif data.dtype == numpy.uint64:
data_name = args.data_name or "segmentation"
else:
sys.stderr.write("Unsupported dtype: {}\n".format(data.dtype))
sys.exit(1)
# Create the new data instance if it doesn't exist already
try:
metadata = VoxelsAccessor.get_metadata(args.hostname, uuid, data_name)
print("Data instance '{}' already exists. Will update.".format( data_name ))
except DVIDException:
print("Creating new data instance: {}".format( data_name ))
metadata = VoxelsMetadata.create_default_metadata(data.shape, data.dtype, 'cxyz', 1.0, 'nanometers')
VoxelsAccessor.create_new(args.hostname, uuid, data_name, metadata)
# Finally, push the data to the server
print("Pushing data to {}".format( '{}/api/node/{}/{}'.format( args.hostname, uuid, data_name ) ))
accessor = VoxelsAccessor(args.hostname, uuid, data_name)
accessor.post_ndarray((0,0,0,0), data.shape, data)
print("DONE.")
def main():
# Read cmd-line args
parser = argparse.ArgumentParser()
parser.add_argument("--hostname", default="localhost:8000")
parser.add_argument(
"--uuid",
required=False,
help=
"The node to upload to. If not provided, a new repo will be created (see --new-repo-alias)."
)
parser.add_argument(
"--data-name",
required=False,
help=
"The name of the data instance to modify. If it doesn't exist, it will be created first."
)
parser.add_argument(
"--new-repo-alias",
required=False,
help="If no uuid is provided, a new repo is created, with this name.")
parser.add_argument("input_file",
help="For example: /tmp/myfile.h5/dataset")
args = parser.parse_args()
if '.h5' not in args.input_file:
sys.stderr.write("File name does not indicate hdf5.\n")
sys.exit(1)
filepath, dset_name = args.input_file.split('.h5')
filepath += '.h5'
if not dset_name:
sys.stderr.write(
"You must provide a dataset name, e.g. myfile.h5/mydataset\n")
sys.exit(1)
if not os.path.exists(filepath):
sys.stderr.write("File doesn't exist: {}\n".format(filepath))
sys.exit(1)
# If no uuid given, create a new repo on the server
uuid = args.uuid
if uuid is None:
alias = args.new_repo_alias or "testrepo"
server = DVIDServerService(args.hostname)
uuid = server.create_new_repo(
alias, "This is a test repo loaded with data from ".format(
args.input_file))
uuid = str(uuid)
# Read the input data from the file
print("Reading {}{}".format(filepath, dset_name))
with h5py.File(filepath) as f_in:
data = f_in[dset_name][:]
# We assume data is 3D or 4D, in C-order
# We adjust it to 4D, fortran-order
if data.ndim == 3:
data = data[..., None]
data = data.transpose()
assert data.flags['F_CONTIGUOUS'], "Data is not contiguous!"
assert data.ndim == 4, "Data must be 3D with axes zyx or 4D with axes zyxc (C-order)"
assert data.shape[
0] == 1, "Data must have exactly 1 channel, not {}".format(
data.shape[0])
# Choose a default data instance name if necessary
if data.dtype == numpy.uint8:
data_name = args.data_name or "grayscale"
elif data.dtype == numpy.uint64:
data_name = args.data_name or "segmentation"
else:
sys.stderr.write("Unsupported dtype: {}\n".format(data.dtype))
sys.exit(1)
# Create the new data instance if it doesn't exist already
try:
metadata = VoxelsAccessor.get_metadata(args.hostname, uuid, data_name)
print("Data instance '{}' already exists. Will update.".format(
data_name))
except DVIDException:
print("Creating new data instance: {}".format(data_name))
metadata = VoxelsMetadata.create_default_metadata(
data.shape, data.dtype, 'cxyz', 1.0, 'nanometers')
VoxelsAccessor.create_new(args.hostname, uuid, data_name, metadata)
# Finally, push the data to the server
print("Pushing data to {}".format('{}/api/node/{}/{}'.format(
args.hostname, uuid, data_name)))
accessor = VoxelsAccessor(args.hostname, uuid, data_name)
accessor.post_ndarray((0, 0, 0, 0), data.shape, data)
print("DONE.")
