def run_export(self): self.progressSignal(0) url = self.NodeDataUrl.value url_path = url.split('://')[1] hostname, api, node, uuid, dataname = url_path.split('/') assert api == 'api' assert node == 'node' axiskeys = self.Input.meta.getAxisKeys() shape = self.Input.meta.shape if self._transpose_axes: axiskeys = reversed(axiskeys) shape = tuple(reversed(shape)) axiskeys = "".join( axiskeys ) if self.OffsetCoord.ready(): offset_start = self.OffsetCoord.value else: offset_start = (0,) * len( self.Input.meta.shape ) self.progressSignal(5) # Get the dataset details try: metadata = VoxelsAccessor.get_metadata(hostname, uuid, dataname) except VoxelsAccessor.BadRequestError as ex: # Dataset doesn't exist yet. Let's create it. metadata = VoxelsMetadata.create_default_metadata( shape, self.Input.meta.dtype, axiskeys, 0.0, "" ) VoxelsAccessor.create_new(hostname, uuid, dataname, metadata) # Since this class is generally used to push large blocks of data, # we'll be nice and set throttle=True client = VoxelsAccessor( hostname, uuid, dataname, throttle=True ) def handle_block_result(roi, data): # Send it to dvid roi = numpy.asarray(roi) roi += offset_start start, stop = roi if self._transpose_axes: data = data.transpose() start = tuple(reversed(start)) stop = tuple(reversed(stop)) client.post_ndarray( start, stop, data ) requester = BigRequestStreamer( self.Input, roiFromShape( self.Input.meta.shape ) ) requester.resultSignal.subscribe( handle_block_result ) requester.progressSignal.subscribe( self.progressSignal ) requester.execute() self.progressSignal(100)
def run_export(self): self.progressSignal(0) url = self.NodeDataUrl.value url_path = url.split('://')[1] hostname, api, node, uuid, dataname = url_path.split('/') assert api == 'api' assert node == 'node' axiskeys = self.Input.meta.getAxisKeys() shape = self.Input.meta.shape if self._transpose_axes: axiskeys = reversed(axiskeys) shape = tuple(reversed(shape)) axiskeys = "".join(axiskeys) if self.OffsetCoord.ready(): offset_start = self.OffsetCoord.value else: offset_start = (0, ) * len(self.Input.meta.shape) self.progressSignal(5) # Get the dataset details try: metadata = VoxelsAccessor.get_metadata(hostname, uuid, dataname) except DVIDException as ex: if ex.status != 404: raise # Dataset doesn't exist yet. Let's create it. metadata = VoxelsMetadata.create_default_metadata( shape, self.Input.meta.dtype, axiskeys, 0.0, "") VoxelsAccessor.create_new(hostname, uuid, dataname, metadata) # Since this class is generally used to push large blocks of data, # we'll be nice and set throttle=True client = VoxelsAccessor(hostname, uuid, dataname, throttle=True) def handle_block_result(roi, data): # Send it to dvid roi = numpy.asarray(roi) roi += offset_start start, stop = roi if self._transpose_axes: data = data.transpose() start = tuple(reversed(start)) stop = tuple(reversed(stop)) client.post_ndarray(start, stop, data) requester = BigRequestStreamer(self.Input, roiFromShape(self.Input.meta.shape)) requester.resultSignal.subscribe(handle_block_result) requester.progressSignal.subscribe(self.progressSignal) requester.execute() self.progressSignal(100)
def _update_subvol_widget(self, node_uuid, dataname, typename): """ Update the subvolume widget with the min/max extents of the given node and dataname. Note: The node and dataname do not necessarily have to match the currently selected node and dataname. This enables the right-click behavior, which can be used to limit your data volume to the size of a different data volume. """ error_msg = None try: if typename == "roi": node_service = DVIDNodeService(self._hostname, str(node_uuid)) roi_blocks_zyx = numpy.array( node_service.get_roi(str(dataname))) maxindex = tuple(DVID_BLOCK_WIDTH * (1 + numpy.max(roi_blocks_zyx, axis=0))) minindex = (0, 0, 0) # Rois are always 3D axiskeys = "zyx" # If the current selection is a dataset, then include a channel dimension if self.get_selection().typename != "roi": axiskeys = "zyxc" minindex = minindex + (0, ) maxindex = maxindex + ( 1, ) # FIXME: This assumes that the selected data has only 1 channel... else: # Query the server raw_metadata = VoxelsAccessor.get_metadata( self._hostname, node_uuid, dataname) voxels_metadata = VoxelsMetadata(raw_metadata) maxindex = voxels_metadata.shape minindex = voxels_metadata.minindex axiskeys = voxels_metadata.axiskeys # If the current selection is a roi, then remove the channel dimension if self.get_selection().typename == "roi": axiskeys = "zyx" minindex = minindex[:-1] maxindex = maxindex[:-1] except (DVIDException, ErrMsg) as ex: error_msg = str(ex) log_exception(logger) else: self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True) if error_msg: self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False) QMessageBox.critical(self, "DVID Error", error_msg) self._subvol_widget.initWithExtents("", (), (), ()) return self._subvol_widget.initWithExtents(axiskeys, maxindex, minindex, maxindex)
def _update_subvol_widget(self, node_uuid, dataname, typename): """ Update the subvolume widget with the min/max extents of the given node and dataname. Note: The node and dataname do not necessarily have to match the currently selected node and dataname. This enables the right-click behavior, which can be used to limit your data volume to the size of a different data volume. """ error_msg = None try: if typename == "roi": node_service = DVIDNodeService(self._hostname, str(node_uuid)) roi_blocks_xyz = numpy.array(node_service.get_roi(str(dataname))) maxindex = tuple(DVID_BLOCK_WIDTH * (1 + numpy.max(roi_blocks_xyz, axis=0))) minindex = (0, 0, 0) # Rois are always 3D axiskeys = "xyz" # If the current selection is a dataset, then include a channel dimension if self.get_selection().typename != "roi": axiskeys = "cxyz" minindex = (0,) + minindex maxindex = (1,) + maxindex # FIXME: This assumes that the selected data has only 1 channel... else: # Query the server raw_metadata = VoxelsAccessor.get_metadata(self._hostname, node_uuid, dataname) voxels_metadata = VoxelsMetadata(raw_metadata) maxindex = voxels_metadata.shape minindex = voxels_metadata.minindex axiskeys = voxels_metadata.axiskeys # If the current selection is a roi, then remove the channel dimension if self.get_selection().typename == "roi": axiskeys = "xyz" minindex = minindex[1:] maxindex = maxindex[1:] except (DVIDException, ErrMsg) as ex: error_msg = str(ex) log_exception(logger) else: self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True) if error_msg: self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False) QMessageBox.critical(self, "DVID Error", error_msg) self._subvol_widget.initWithExtents("", (), (), ()) return self._subvol_widget.initWithExtents(axiskeys, maxindex, minindex, maxindex)
def _update_display(self): super(DvidDataSelectionBrowser, self)._update_display() hostname, dset_uuid, dataname, node_uuid = self.get_selection() enable_contents = self._repos_info is not None and dataname != "" and node_uuid != "" self._roi_groupbox.setEnabled(enable_contents) if not dataname or not node_uuid: self._roi_widget.initWithExtents("", (), (), ()) return error_msg = None try: # Query the server raw_metadata = VoxelsAccessor.get_metadata(hostname, node_uuid, dataname) voxels_metadata = VoxelsMetadata(raw_metadata) except DVIDException as ex: error_msg = ex.message except ErrMsg as ex: error_msg = str(ErrMsg) except VoxelsAccessor.BadRequestError as ex: # DVID will return an error if the selected dataset # isn't a 'voxels' dataset and thus has no voxels metadata self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False) return else: self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True) if error_msg: QMessageBox.critical(self, "DVID Error", error_msg) self._roi_widget.initWithExtents("", (), (), ()) return self._roi_widget.initWithExtents(voxels_metadata.axiskeys, voxels_metadata.shape, voxels_metadata.minindex, voxels_metadata.shape)
def _update_display(self): super(DvidDataSelectionBrowser, self)._update_display() hostname, dset_uuid, dataname, node_uuid = self.get_selection() enable_contents = self._repos_info is not None and dataname != "" and node_uuid != "" self._roi_groupbox.setEnabled(enable_contents) if not dataname or not node_uuid: self._roi_widget.initWithExtents("", (), (), ()) return error_msg = None try: # Query the server raw_metadata = VoxelsAccessor.get_metadata(hostname, node_uuid, dataname) voxels_metadata = VoxelsMetadata(raw_metadata) except DVIDException as ex: error_msg = ex.message except ErrMsg as ex: error_msg = str(ErrMsg) except VoxelsAccessor.BadRequestError as ex: # DVID will return an error if the selected dataset # isn't a 'voxels' dataset and thus has no voxels metadata self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False) return else: self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True) if error_msg: QMessageBox.critical(self, "DVID Error", error_msg) self._roi_widget.initWithExtents("", (), (), ()) return self._roi_widget.initWithExtents( voxels_metadata.axiskeys, voxels_metadata.shape, voxels_metadata.minindex, voxels_metadata.shape )
def main(): # Read cmd-line args parser = argparse.ArgumentParser() parser.add_argument("--hostname", default="localhost:8000") parser.add_argument("--uuid", required=False, help="The node to upload to. If not provided, a new repo will be created (see --new-repo-alias).") parser.add_argument("--data-name", required=False, help="The name of the data instance to modify. If it doesn't exist, it will be created first.") parser.add_argument("--new-repo-alias", required=False, help="If no uuid is provided, a new repo is created, with this name.") parser.add_argument("input_file", help="For example: /tmp/myfile.h5/dataset") args = parser.parse_args() if '.h5' not in args.input_file: sys.stderr.write("File name does not indicate hdf5.\n") sys.exit(1) filepath, dset_name = args.input_file.split('.h5') filepath += '.h5' if not dset_name: sys.stderr.write("You must provide a dataset name, e.g. myfile.h5/mydataset\n") sys.exit(1) if not os.path.exists(filepath): sys.stderr.write("File doesn't exist: {}\n".format(filepath)) sys.exit(1) # If no uuid given, create a new repo on the server uuid = args.uuid if uuid is None: alias = args.new_repo_alias or "testrepo" server = DVIDServerService(args.hostname) uuid = server.create_new_repo(alias, "This is a test repo loaded with data from ".format(args.input_file)) uuid = str(uuid) # Read the input data from the file print("Reading {}{}".format( filepath, dset_name )) with h5py.File(filepath) as f_in: data = f_in[dset_name][:] # We assume data is 3D or 4D, in C-order # We adjust it to 4D, fortran-order if data.ndim == 3: data = data[...,None] data = data.transpose() assert data.flags['F_CONTIGUOUS'], "Data is not contiguous!" assert data.ndim == 4, "Data must be 3D with axes zyx or 4D with axes zyxc (C-order)" assert data.shape[0] == 1, "Data must have exactly 1 channel, not {}".format( data.shape[0] ) # Choose a default data instance name if necessary if data.dtype == numpy.uint8: data_name = args.data_name or "grayscale" elif data.dtype == numpy.uint64: data_name = args.data_name or "segmentation" else: sys.stderr.write("Unsupported dtype: {}\n".format(data.dtype)) sys.exit(1) # Create the new data instance if it doesn't exist already try: metadata = VoxelsAccessor.get_metadata(args.hostname, uuid, data_name) print("Data instance '{}' already exists. Will update.".format( data_name )) except DVIDException: print("Creating new data instance: {}".format( data_name )) metadata = VoxelsMetadata.create_default_metadata(data.shape, data.dtype, 'cxyz', 1.0, 'nanometers') VoxelsAccessor.create_new(args.hostname, uuid, data_name, metadata) # Finally, push the data to the server print("Pushing data to {}".format( '{}/api/node/{}/{}'.format( args.hostname, uuid, data_name ) )) accessor = VoxelsAccessor(args.hostname, uuid, data_name) accessor.post_ndarray((0,0,0,0), data.shape, data) print("DONE.")
def main(): # Read cmd-line args parser = argparse.ArgumentParser() parser.add_argument("--hostname", default="localhost:8000") parser.add_argument( "--uuid", required=False, help= "The node to upload to. If not provided, a new repo will be created (see --new-repo-alias)." ) parser.add_argument( "--data-name", required=False, help= "The name of the data instance to modify. If it doesn't exist, it will be created first." ) parser.add_argument( "--new-repo-alias", required=False, help="If no uuid is provided, a new repo is created, with this name.") parser.add_argument("input_file", help="For example: /tmp/myfile.h5/dataset") args = parser.parse_args() if '.h5' not in args.input_file: sys.stderr.write("File name does not indicate hdf5.\n") sys.exit(1) filepath, dset_name = args.input_file.split('.h5') filepath += '.h5' if not dset_name: sys.stderr.write( "You must provide a dataset name, e.g. myfile.h5/mydataset\n") sys.exit(1) if not os.path.exists(filepath): sys.stderr.write("File doesn't exist: {}\n".format(filepath)) sys.exit(1) # If no uuid given, create a new repo on the server uuid = args.uuid if uuid is None: alias = args.new_repo_alias or "testrepo" server = DVIDServerService(args.hostname) uuid = server.create_new_repo( alias, "This is a test repo loaded with data from ".format( args.input_file)) uuid = str(uuid) # Read the input data from the file print("Reading {}{}".format(filepath, dset_name)) with h5py.File(filepath) as f_in: data = f_in[dset_name][:] # We assume data is 3D or 4D, in C-order # We adjust it to 4D, fortran-order if data.ndim == 3: data = data[..., None] data = data.transpose() assert data.flags['F_CONTIGUOUS'], "Data is not contiguous!" assert data.ndim == 4, "Data must be 3D with axes zyx or 4D with axes zyxc (C-order)" assert data.shape[ 0] == 1, "Data must have exactly 1 channel, not {}".format( data.shape[0]) # Choose a default data instance name if necessary if data.dtype == numpy.uint8: data_name = args.data_name or "grayscale" elif data.dtype == numpy.uint64: data_name = args.data_name or "segmentation" else: sys.stderr.write("Unsupported dtype: {}\n".format(data.dtype)) sys.exit(1) # Create the new data instance if it doesn't exist already try: metadata = VoxelsAccessor.get_metadata(args.hostname, uuid, data_name) print("Data instance '{}' already exists. Will update.".format( data_name)) except DVIDException: print("Creating new data instance: {}".format(data_name)) metadata = VoxelsMetadata.create_default_metadata( data.shape, data.dtype, 'cxyz', 1.0, 'nanometers') VoxelsAccessor.create_new(args.hostname, uuid, data_name, metadata) # Finally, push the data to the server print("Pushing data to {}".format('{}/api/node/{}/{}'.format( args.hostname, uuid, data_name))) accessor = VoxelsAccessor(args.hostname, uuid, data_name) accessor.post_ndarray((0, 0, 0, 0), data.shape, data) print("DONE.")