Exemplo n.º 1
0
def get_synapses_dvid(dvid_server, dvid_uuid, dvid_annot, dvid_roi):
    dvid_node = DVIDNodeService(dvid_server, dvid_uuid,
                                'flyem_syn_eval','flyem_syn_eval')
    synapses_json = dvid_node.custom_request(
        '%s/roi/%s' % (dvid_annot, dvid_roi),
        None, ConnectionMethod.GET)
    return synapses_json.decode()
Exemplo n.º 2
0
        def writer(element_pairs):
            from libdvid import DVIDNodeService, Vertex, Edge

            # write graph information
            node_service = DVIDNodeService(str(server), str(uuid))

            if element_pairs is None:
                return

            vertices = []
            edges = []
            for element_pair in element_pairs:
                edge, weight = element_pair
                v1, v2 = edge

                if v2 == -1:
                    vertices.append(Vertex(v1, weight))
                else:
                    edges.append(Edge(v1, v2, weight))

            if len(vertices) > 0:
                node_service.update_vertices(str(graph_name), vertices)

            if len(edges) > 0:
                node_service.update_edges(str(graph_name), edges)

            return []
Exemplo n.º 3
0
    def _init_version(self, uuid):
        # create connection to repo
        self.nodeconn = DVIDNodeService(str(self.server), str(uuid))
        
        # fetch all repo information
        status, data, errmsg = self.rawconn.make_request("/repo/" + uuid + "/info", ConnectionMethod.GET)
        self.repoinfo = json.loads(data)
        self.allinstances = {}
        # load all versions in repo
        self.alluuids = set()
        dag = self.repoinfo["DAG"]["Nodes"]
        for uuidt, nodedata in dag.items():
            self.alluuids.add(str(nodedata["UUID"]))
        
        for instancename, val in self.repoinfo["DataInstances"].items():
            # name is not necessarily unique to a repo
            self.allinstances[(str(instancename), str(val["Base"]["DataUUID"]))] = str(val["Base"]["TypeName"])
        

        # datainstances that should be hidden (array of names) 
        try:
            self.hidden_instances = set(self.nodeconn.get_json(self.MetaLocation, self.RestrictionName))
        except:
            self.hidden_instances = set()
        
        nodeids = {}
        # check if locked note
        dag = self.repoinfo["DAG"]["Nodes"]
        for tuuid, nodedata in dag.items():
            nodeids[str(nodedata["VersionID"])] = nodedata
            if tuuid.startswith(uuid):
                self.uuid = str(tuuid)
                self.current_node = nodedata
                self.locked = nodedata["Locked"]
           
        # load all ancestors
        ancestors = set()

        # commit history uuid, commit note in order from oldest to newest
        self.loghistory = []
        currnode = self.current_node
        while True:
            ancestors.add(str(currnode["UUID"]))
            self.loghistory.append((str(currnode["UUID"]), currnode["Note"]))
            if len(currnode["Parents"]) > 0:
                currnode = nodeids[str(currnode["Parents"][0])] 
            else:
                break
        self.loghistory.reverse()

        # load all instances
        self.activeinstances = {}
        
        if not self.locked:
            tempuuid = self.loghistory[-1][0]
            self.loghistory[-1] = (tempuuid, "(open node)")

        for instancename, val in self.repoinfo["DataInstances"].items():
            if str(val["Base"]["RepoUUID"]) in ancestors:
                self.activeinstances[str(instancename)] = str(val["Base"]["TypeName"])
Exemplo n.º 4
0
 def get_labels2():
     # fetch second label volume
     node_service2 = DVIDNodeService(str(server2), str(uuid2))
     # retrieve data from roi start position
     return node_service2.get_labels3D(
         str(label_name2), (size1, size2, size3), (subvolume.roi[0], subvolume.roi[1], subvolume.roi[2])
     )
Exemplo n.º 5
0
    def setUpClass(cls):
        """
        Override.  Called by nosetests.
        """
        # Choose names
        cls.dvid_repo = "datasetA"
        cls.data_name = "random_data"
        cls.volume_location = "/repos/{dvid_repo}/volumes/{data_name}".format( **cls.__dict__ )

        cls.data_uuid = get_testrepo_root_uuid()
        cls.node_location = "/repos/{dvid_repo}/nodes/{data_uuid}".format( **cls.__dict__ )

        # Generate some test data
        #data = numpy.random.randint(0, 255, (128, 256, 512, 1)).astype( numpy.uint8 )
        
        data = numpy.zeros((128, 256, 512, 1), dtype=numpy.uint8)
        data.flat[:] = numpy.arange( numpy.prod((128, 256, 512, 1)) )
        cls.original_data = data
        cls.voxels_metadata = VoxelsMetadata.create_default_metadata(data.shape, data.dtype, "zyxc", 1.0, "")

        # Write it to a new data instance
        node_service = DVIDNodeService(TEST_DVID_SERVER, cls.data_uuid)

        node_service.create_grayscale8(cls.data_name)
        node_service.put_gray3D( cls.data_name, data[...,0], (0,0,0) )
Exemplo n.º 6
0
def tbars_push_dvid(tbars_json, dvid_server, dvid_uuid, dvid_annot):
    dvid_node = DVIDNodeService(dvid_server, dvid_uuid, 'fpl', 'fpl')
    dvid_conn = DVIDConnection(dvid_server, 'fpl', 'fpl')

    # create annotation if necessary
    try:
        dvid_node.custom_request('%s/info' % dvid_annot, None,
                                 ConnectionMethod.GET)
    except DVIDException as e:
        post_json = json.dumps({
            'typename': 'annotation',
            'dataname': dvid_annot
        })
        status, body, error_message = dvid_conn.make_request(
            '/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
            post_json.encode('utf-8'))

    num_at_once = 100000
    n_tot = len(tbars_json)
    for ii in range(0, n_tot, num_at_once):
        jj = np.minimum(ii + num_at_once, n_tot)
        data = json.dumps(tbars_json[ii:jj])
        oo = dvid_node.custom_request('%s/elements' % dvid_annot,
                                      data.encode('utf-8'),
                                      ConnectionMethod.POST)
Exemplo n.º 7
0
def retrieve_node_service(server, uuid, resource_server, resource_port, appname="sparkservices"):
    """Create a DVID node service object"""

    server = str(server)  
   
    # refresh dvid server meta if localhost (since it is exclusive or points to global db)
    """
    if server.startswith("http://127.0.0.1") or  \
            server.startswith("http://localhost") or  \
            server.startswith("127.0.0.1") or server.startswith("localhost"):
        
        import os
        if not os.path.exists("/tmp/reloaded.hack"):
            addr = server + "/api/server/reload-metadata"
            if not server.startswith("http://"):
                addr = "http://" + addr

            session = default_dvid_session()
            session.post(addr)
            open("/tmp/reloaded.hack", 'w').close()
    """

    from libdvid import DVIDNodeService
    import os
    username = os.environ["USER"]

    if resource_server != "":
        node_service = DVIDNodeService(server, str(uuid), username, appname, str(resource_server), resource_port)
    else:
        node_service = DVIDNodeService(server, str(uuid), username, appname)


    return node_service
Exemplo n.º 8
0
def get_blocksize(dvid_server, uuid, dataname):
    """Gets block size for supplied data name.

    Note:
        Does not check for existence of body and whether
        it is an array type.  The block size is always
        assumes to be isotropic.
   
    Args:
        dvid_server (str): location of dvid server
        uuid (str): version id
        dataname (str): data instance
   
    Returns:
        (z,y,x) blocksize.

    Raises:
        DVIDExceptions are not caught in this function and will be
        transferred to the caller.
    """

    ns = DVIDNodeService(str(dvid_server), str(uuid))
    info = ns.get_typeinfo(dataname)
    x, y, z = info["Extended"]["BlockSize"]  # DVID ordered x,y,z
    return (z, y, x)
Exemplo n.º 9
0
def get_blocksize(dvid_server, uuid, dataname):
    """Gets block size for supplied data name.

    Note:
        Does not check for existence of body and whether
        it is an array type.  The block size is always
        assumes to be isotropic.
   
    Args:
        dvid_server (str): location of dvid server
        uuid (str): version id
        dataname (str): data instance
   
    Returns:
        (z,y,x) blocksize.

    Raises:
        DVIDExceptions are not caught in this function and will be
        transferred to the caller.
    """

    ns = DVIDNodeService(str(dvid_server), str(uuid))
    info = ns.get_typeinfo(dataname)
    x,y,z = info["Extended"]["BlockSize"] # DVID ordered x,y,z
    return (z,y,x)
Exemplo n.º 10
0
class IoDVID(object):
    def __init__(self, server_address, uuid, key):
        assert WITH_DVID, "Need dvid"
        self.ds = DVIDNodeService(server_address, uuid)
        self.key = key

        # get the shape the dvid way...
        endpoint = "/" + self.key + "/info"
        attributes = self.ds.custom_request(endpoint, "", ConnectionMethod.GET)
        # TODO do we need to increase by 1 here ?
        self._shape = tuple(mp + 1 for mp in attributes["MaxPoint"])

    def read(self, bb):
        offset = tuple(b.start for b in bb)
        shape = tuple(b.stop - b.start for b in bb)
        return self.ds.get_gray3D(self.key, shape, offset)

    def write(self, out, out_bb):
        raise NotImplementedError("Writing to DVID is not yet implemented!")

    @property
    def shape(self):
        return self._shape

    def close(self):
        pass
Exemplo n.º 11
0
 def get_segmask():
     node_service = DVIDNodeService(str(pdconf["dvid-server"]), str(pdconf["uuid"]))
     # retrieve data from roi start position
     return node_service.get_labels3D(
         str(pdconf["segmentation-name"]),
         (size1, size2, size3),
         (subvolume.roi[0] - border, subvolume.roi[1] - border, subvolume.roi[2] - border),
     )
Exemplo n.º 12
0
    def test_custom_request(self):
        node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
                                       "*****@*****.**", "test_custom_app")
        response_body = node_service.custom_request("log", b"",
                                                    ConnectionMethod.GET)

        # This shouldn't raise an exception
        json_data = json.loads(response_body)
Exemplo n.º 13
0
 def create_new( hostname, uuid, data_name, voxels_metadata ):
     node_service = DVIDNodeService(hostname, uuid)
     typename = voxels_metadata.determine_dvid_typename()
     if typename == "grayscale8":
         node_service.create_grayscale8(data_name)
     elif typename == "labels64":
         node_service.create_labelblk("test_labels_3d")
     else:
         raise RuntimeError("Don't know how to create a data instance of type '{}'".format(typename))
 def storeFeaturesForFrame(self, features, timeframe):
     """
     Stores feature data
     """
     assert(self.server_address is not None)
     assert(self.uuid is not None)
     node_service = DVIDNodeService(self.server_address, self.uuid)
     node_service.create_keyvalue(self.keyvalue_store)
     node_service.put(self.keyvalue_store, "frame-{}".format(timeframe), json.dumps(features))
Exemplo n.º 15
0
 def loadFeaturesForFrame(self, features, timeframe):
     """
     loads feature data
     """
     assert(self.server_address is not None)
     assert(self.uuid is not None)
     node_service = DVIDNodeService(self.server_address, self.uuid)
     node_service.create_keyvalue(self.keyvalue_store)
     return json.loads(node_service.get(self.keyvalue_store, "frame-{}".format(timeframe)))
Exemplo n.º 16
0
    def __init__(self, server_address, uuid, key):
        assert WITH_DVID, "Need dvid"
        self.ds = DVIDNodeService(server_address, uuid)
        self.key = key

        # get the shape the dvid way...
        endpoint = "/" + self.key + "/info"
        attributes = self.ds.custom_request(endpoint, "", ConnectionMethod.GET)
        # TODO do we need to increase by 1 here ?
        self._shape = tuple(mp + 1 for mp in attributes["MaxPoint"])
Exemplo n.º 17
0
 def getTimeRange(self, Resource, PathInResource):
     """
     Count Label images to derive the total number of frames
     Loads label image data from local resource file in hdf5 format.
     PathInResource provides the internal image path 
     Return tuple of (first frame, last frame)
     """
     node_service = DVIDNodeService(Resource, PathInResource)
     config = json.loads(node_service.get("config", "imageInfo"))
     return config["time_range"]
Exemplo n.º 18
0
 def get_labels():
     # extract labels 64
     node_service = DVIDNodeService(str(server), str(uuid))
     # retrieve data from roi start position
     return node_service.get_labels3D(
         str(label_name),
         (size1, size2, size3),
         (subvolume.roi[0], subvolume.roi[1], subvolume.roi[2]),
         roi=str(roiname),
     )
Exemplo n.º 19
0
    def test_copy_roi(self):
        expected_blocks = [(1,2,3),(2,3,4),(4,5,6)]

        # We aren't really exercising transferring between servers here.  Too bad.
        src_info  = (TEST_DVID_SERVER, self.uuid, 'src_roi')
        dest_info = (TEST_DVID_SERVER, self.uuid, 'dest_roi')

        node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid, "unittest@test_roi_utils.py", "Test_roi_utils")
        written_blocks = node_service.get_roi("src_roi")
        assert set(expected_blocks) == set(written_blocks)
Exemplo n.º 20
0
 def getTimeRange(self, Resource, PathInResource):
     """
     Count Label images to derive the total number of frames
     Loads label image data from local resource file in hdf5 format.
     PathInResource provides the internal image path 
     Return tuple of (first frame, last frame)
     """
     node_service = DVIDNodeService(Resource, PathInResource)
     config = json.loads(node_service.get("config", "imageInfo"))
     return config["time_range"]
Exemplo n.º 21
0
    def execute(self):
        from DVIDSparkServices.reconutils import SimpleGraph
        from libdvid import DVIDNodeService
        from pyspark import SparkContext
        from pyspark import StorageLevel

        if "chunk-size" in self.config_data["options"]:
            self.chunksize = self.config_data["options"]["chunk-size"]

        #  grab ROI
        distrois = self.sparkdvid_context.parallelize_roi(self.config_data["dvid-info"]["roi"], self.chunksize)

        num_partitions = distrois.getNumPartitions()

        # map ROI to label volume (1 pixel overlap)
        label_chunks = self.sparkdvid_context.map_labels64(
            distrois, self.config_data["dvid-info"]["label-name"], 1, self.config_data["dvid-info"]["roi"]
        )

        # map labels to graph data -- external program (eventually convert neuroproof metrics and graph to a python library) ?!
        sg = SimpleGraph.SimpleGraph(self.config_data["options"])

        # extract graph
        graph_elements = label_chunks.flatMap(sg.build_graph)

        # group data for vertices and edges
        graph_elements_red = graph_elements.reduceByKey(lambda a, b: a + b)

        # repartition by first vertex to better group edges together
        graph_elements_red = graph_elements_red.partitionBy(num_partitions, lambda a: hash(a[0]))

        graph_elements_red.persist(StorageLevel.MEMORY_ONLY)  # ??
        graph_vertices = graph_elements_red.filter(sg.is_vertex)
        graph_edges = graph_elements_red.filter(sg.is_edge)

        # create graph
        node_service = DVIDNodeService(
            str(self.config_data["dvid-info"]["dvid-server"]), str(self.config_data["dvid-info"]["uuid"])
        )

        node_service.create_graph(str(self.config_data["dvid-info"]["graph-name"]))

        # dump graph -- should this be wrapped through utils or through sparkdvid ??
        # will this result in too many request (should they be accumulated) ??
        # currently looking at one partitioning at a time to try to group requests
        self.sparkdvid_context.foreachPartition_graph_elements(
            graph_vertices, self.config_data["dvid-info"]["graph-name"]
        )
        self.sparkdvid_context.foreachPartition_graph_elements(graph_edges, self.config_data["dvid-info"]["graph-name"])

        if "debug" in self.config_data["options"] and self.config_data["options"]["debug"]:
            num_elements = graph_elements.count()
            print "DEBUG: ", num_elements

        graph_elements_red.unpersist()
Exemplo n.º 22
0
 def get_labels():
     # extract labels 64
     node_service = DVIDNodeService(str(server), str(uuid))
     # retrieve data from roi start position considering border
     return node_service.get_labels3D(
         str(label_name),
         (size1, size2, size3),
         (subvolume.roi[0] - border, subvolume.roi[1] - border, subvolume.roi[2] - border),
         compress=True,
         roi=str(roiname),
     )
Exemplo n.º 23
0
 def put_labels():
     node_service = DVIDNodeService(str(server), str(uuid))
     # send data from roi start position
     node_service.put_labels3D(
         str(label_name),
         seg,
         (subvolume.roi.x1, subvolume.roi.y1, subvolume.roi.z1),
         compress=True,
         roi=str(roi_name),
         mutate=mutate,
     )
Exemplo n.º 24
0
 def get_gray():
     node_service = DVIDNodeService(str(server), str(uuid))
     return node_service.get_gray3D(
         str(gray_name),
         (size1, size2, size3),
         (
             subvolume.roi.x1 - subvolume.border,
             subvolume.roi.y1 - subvolume.border,
             subvolume.roi.z1 - subvolume.border,
         ),
     )
Exemplo n.º 25
0
    def test_isdatainstance(self):
        """Tests is_datainstance function.
        """

        service = DVIDServerService(dvidserver)
        uuid = service.create_new_repo("foo", "bar")

        ns = DVIDNodeService(dvidserver, uuid)
        ns.create_labelblk("labels")

        self.assertTrue(is_datainstance(dvidserver, uuid, "labels"))
        self.assertFalse(is_datainstance(dvidserver, uuid, "labels2"))
Exemplo n.º 26
0
    def getImageShape(self, Resource, PathInResource):
        """
        Derive Image Shape from label image.
        Loads label image data from local resource file in hdf5 format.
        PathInResource provides the internal image path 
        Return list with image dimensions
        """

        node_service = DVIDNodeService(Resource, PathInResource)
        config = json.loads(node_service.get("config", "imageInfo"))
        self.shape =  config["shape"]
        return self.shape
Exemplo n.º 27
0
def get_output_vol(index):
    output_service = DVIDNodeService(
        str(config['outputs'][index]['dvid']['server']),
        str(config['outputs'][index]['dvid']['uuid']))
    output_name = config['outputs'][index]['dvid']['segmentation-name']
    output_bb_xyz = config['outputs'][index]['geometry']['bounding-box']
    output_bb_zyx = np.array(output_bb_xyz)[:, ::-1]
    output_shape = output_bb_zyx[1] - output_bb_zyx[0]

    output_volume = output_service.get_labels3D(output_name, output_shape,
                                                output_bb_zyx[0])
    return output_volume
Exemplo n.º 28
0
def copy_roi(src_info, dest_info):
    src_service = DVIDNodeService(src_info.server, src_info.uuid)
    dest_service = DVIDNodeService(dest_info.server, dest_info.uuid)

    # If necessary, create the ROI on the destination server
    try:
        info = dest_service.get_typeinfo(dest_info.name)
    except DVIDException:
        dest_service.create_roi(dest_info.name)

    roi_blocks = src_service.get_roi(src_info.name)
    dest_service.post_roi(dest_info.name, roi_blocks)
Exemplo n.º 29
0
    def getImageShape(self, Resource, PathInResource):
        """
        Derive Image Shape from label image.
        Loads label image data from local resource file in hdf5 format.
        PathInResource provides the internal image path 
        Return list with image dimensions
        """

        node_service = DVIDNodeService(Resource, PathInResource)
        config = json.loads(node_service.get("config", "imageInfo"))
        self.shape = config["shape"]
        return self.shape
Exemplo n.º 30
0
    def test_copy_roi(self):
        expected_blocks = [(1, 2, 3), (2, 3, 4), (4, 5, 6)]

        # We aren't really exercising transferring between servers here.  Too bad.
        src_info = (TEST_DVID_SERVER, self.uuid, 'src_roi')
        dest_info = (TEST_DVID_SERVER, self.uuid, 'dest_roi')

        node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
                                       "unittest@test_roi_utils.py",
                                       "Test_roi_utils")
        written_blocks = node_service.get_roi("src_roi")
        assert set(expected_blocks) == set(written_blocks)
Exemplo n.º 31
0
    def test_keyvalue(self):
        node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
        node_service.create_keyvalue("test_keyvalue")
        node_service.put("test_keyvalue", "kkkk", "vvvv")
        readback_value = node_service.get("test_keyvalue", "kkkk")
        self.assertEqual(readback_value, "vvvv")
 
        with self.assertRaises(RuntimeError):
            node_service.put("test_keyvalue", "kkkk", 123) # 123 is not a buffer.
Exemplo n.º 32
0
    def _update_subvol_widget(self, node_uuid, dataname, typename):
        """
        Update the subvolume widget with the min/max extents of the given node and dataname.
        Note: The node and dataname do not necessarily have to match the currently
              selected node and dataname.
              This enables the right-click behavior, which can be used to
              limit your data volume to the size of a different data volume.
        """
        error_msg = None
        try:
            if typename == "roi":
                node_service = DVIDNodeService(self._hostname, str(node_uuid))
                roi_blocks_zyx = numpy.array(
                    node_service.get_roi(str(dataname)))
                maxindex = tuple(DVID_BLOCK_WIDTH *
                                 (1 + numpy.max(roi_blocks_zyx, axis=0)))
                minindex = (0, 0, 0)  # Rois are always 3D
                axiskeys = "zyx"
                # If the current selection is a dataset, then include a channel dimension
                if self.get_selection().typename != "roi":
                    axiskeys = "zyxc"
                    minindex = minindex + (0, )
                    maxindex = maxindex + (
                        1,
                    )  # FIXME: This assumes that the selected data has only 1 channel...
            else:
                # Query the server
                raw_metadata = VoxelsAccessor.get_metadata(
                    self._hostname, node_uuid, dataname)
                voxels_metadata = VoxelsMetadata(raw_metadata)
                maxindex = voxels_metadata.shape
                minindex = voxels_metadata.minindex
                axiskeys = voxels_metadata.axiskeys
                # If the current selection is a roi, then remove the channel dimension
                if self.get_selection().typename == "roi":
                    axiskeys = "zyx"
                    minindex = minindex[:-1]
                    maxindex = maxindex[:-1]
        except (DVIDException, ErrMsg) as ex:
            error_msg = str(ex)
            log_exception(logger)
        else:
            self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True)

        if error_msg:
            self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False)
            QMessageBox.critical(self, "DVID Error", error_msg)
            self._subvol_widget.initWithExtents("", (), (), ())
            return

        self._subvol_widget.initWithExtents(axiskeys, maxindex, minindex,
                                            maxindex)
Exemplo n.º 33
0
    def getImageDataAtTimeFrame(self, Resource, PathInResource, axes, timeframe):
        """
        Loads image data from local resource file in hdf5 format.
        PathInResource provides the internal image path 
        Return numpy array of image data at timeframe.
        """
        node_service = DVIDNodeService(Resource, PathInResource)

        if (self.shape == None):
            self.getImageShape(Resource, PathInResource)

        raw_frame = node_service.get_gray3D(self._getRawImageName(timeframe), tuple(self.shape), (0,0,0))
        return raw_frame
Exemplo n.º 34
0
    def __init__(self, hostname, uuid, data_name, 
                 query_args=None, 
                 throttle=None, 
                 retry_timeout=60.0, 
                 retry_interval=1.0, 
                 warning_interval=30.0, 
                 _metadata=None,
                 _access_type="raw"):
        """
        :param uuid: The node uuid
        :param data_name: The name of the volume
        :param hostname: The DVID server hostname e.g. 'localhost:8000'
        :param throttle: Enable the DVID 'throttle' flag for all get/post requests
        :param retry_timeout: Total time to spend repeating failed requests before giving up.
                              (Set to 0 to prevent retries.)
        :param retry_interval: Time to wait before repeating a failed get/post.
        :param warning_interval: If the retry period exceeds this interval (but hasn't 
                                 hit the retry_timeout yet), a warning is emitted.
        :param _metadata: If provided, used as the metadata for the accessor.  Otherwise, the server is queried to obtain this volume's metadata.
        
        .. note:: When DVID is overloaded, it may indicate its busy status by returning a ``503`` 
                  (service unavailable) error in response to a get/post request.  In that case, 
                  the get/post methods below will automatically repeat the failed request until 
                  the `retry_timeout` is reached.
        """
        self.hostname = hostname
        self.uuid = uuid
        self.data_name = data_name
        self._retry_timeout = retry_timeout
        self._retry_interval = retry_interval
        self._warning_interval = warning_interval
        self._query_args = query_args or {}
        self._access_type = _access_type
        self._node_service = DVIDNodeService(hostname, uuid)
        
        # Special case: throttle can be set explicity via the keyword or implicitly via the query_args.
        # Make sure they are consistent.
        if 'throttle' in self._query_args:
            if self._query_args['throttle'] == 'on':
                assert throttle is None or throttle is True
                self._throttle = True
            if self._query_args['throttle'] == 'off':
                assert throttle is None or throttle is False
                self._throttle = False
        elif throttle is None:
            self._throttle = False
        else:
            self._throttle = throttle

        # Request this volume's metadata from DVID if necessary
        self.voxels_metadata = _metadata or VoxelsAccessor.get_metadata( hostname, uuid, data_name )
Exemplo n.º 35
0
    def ingest_data(self):
        """
        Ideally, this would be run within the __init__ function,
        but operators should never raise non-fatal exceptions within Operator.__init__()
        (See OperatorMetaClass.__call__)
        This serves as an alternative init function, from which we are allowed to raise exceptions.
        """
        try:
            node_service = DVIDNodeService(self._hostname, self._uuid)
            roi_blocks_zyx = numpy.array(node_service.get_roi(self._roi_name))
        except DVIDException as ex:
            if ex.status == http.client.NOT_FOUND:
                raise OpDvidRoi.DatasetReadError("DVIDException: " +
                                                 ex.message)
            raise
        except ErrMsg as ex:
            raise OpDvidRoi.DatasetReadError("ErrMsg: " + str(ex))

        # Store a dense array of the block flags
        # Each voxel of this dataset represents a dvid block, so this volume
        # is 32x smaller than the full-res data in every dimension.
        # TODO: For now, we only respect the high-side of the bounding box, and force the low-side to 0,0,0.
        shape = tuple(1 + numpy.max(roi_blocks_zyx, axis=0))
        slice_shape = (1, ) + shape[1:]
        self._mem_file = h5py.File(str(id(self)),
                                   driver='core',
                                   backing_store=False,
                                   mode='w')
        dset = self._mem_file.create_dataset('data',
                                             shape=shape,
                                             dtype=numpy.uint8,
                                             chunks=slice_shape,
                                             compression='lzf')

        # Allocate temporary array slice for writing into the dataset
        z_slice = numpy.ndarray(slice_shape, numpy.uint8)

        # Group the block indexes by z-coordinate
        # FIXME: This might get slow for lots of blocks because of all the calls to this lambda.
        #        Consider upgrading to cytools.groupby instead
        group_iter = groupby(roi_blocks_zyx,
                             lambda block_index: block_index[0])

        # Set a pixel for each block index, one z-slice at a time
        for z, block_indexes_iter in group_iter:
            block_indexes = numpy.array(list(block_indexes_iter))
            block_indexes = block_indexes[:, 1:]  # drop z index
            z_slice[:] = 0
            z_slice[0][tuple(block_indexes.transpose())] = 1
            dset[z, :, :] = z_slice[0, ...]
        self._dset = dset
Exemplo n.º 36
0
    def getLabelImageForFrame(self, Resource, PathInResource, timeframe):
        """
        Loads label image data from local resource file in hdf5 format.
        PathInResource provides the internal image path 
        Return numpy array of image data at timeframe.
        """

        if (self.shape == None):
            self.getImageShape(Resource, PathInResource)

        node_service = DVIDNodeService(Resource, PathInResource)
        seg_frame = np.array(node_service.get_labels3D(self._getSegmentationName(timeframe),
                            tuple(self.shape), (0,0,0))).astype(np.uint32)
        return seg_frame
Exemplo n.º 37
0
    def test_DataInstance(self):
        """Tests DataInstance class.
        """

        service = DVIDServerService(dvidserver)
        uuid = service.create_new_repo("foo", "bar")

        ns = DVIDNodeService(dvidserver, uuid)
        ns.create_labelblk("labels")
        ns.create_grayscale8("gray")
        ns.create_keyvalue("kv")

        try:
            temp = DataInstance(dvidserver, uuid, "blah")
        except ValueError:
            # correct caught error
            self.assertTrue(True)

        labels = DataInstance(dvidserver, uuid, "labels")
        gray = DataInstance(dvidserver, uuid, "gray")
        kv = DataInstance(dvidserver, uuid, "kv")

        self.assertTrue(labels.is_array())
        self.assertTrue(labels.is_labels())

        self.assertTrue(gray.is_array())
        self.assertFalse(gray.is_labels())

        self.assertFalse(kv.is_array())
        self.assertFalse(kv.is_labels())
Exemplo n.º 38
0
def set_syncs(dvid_server,
              dvid_uuid,
              dvid_annot,
              dvid_segm=None,
              dvid_labelsz=None):
    dvid_node = DVIDNodeService(dvid_server, dvid_uuid, os.getenv('USER'),
                                'fpl')
    dvid_conn = DVIDConnection(dvid_server, os.getenv('USER'), 'fpl')

    # create annotation if necessary
    try:
        dvid_node.custom_request('%s/info' % dvid_annot, None,
                                 ConnectionMethod.GET)
    except DVIDException as e:
        post_json = json.dumps({
            'typename': 'annotation',
            'dataname': dvid_annot
        })
        status, body, error_message = dvid_conn.make_request(
            '/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
            post_json.encode('utf-8'))

    if dvid_labelsz is not None:
        # create labelsz if necessary
        try:
            dvid_node.custom_request('%s/info' % dvid_labelsz, None,
                                     ConnectionMethod.GET)
        except DVIDException as e:
            post_json = json.dumps({
                'typename': 'labelsz',
                'dataname': dvid_labelsz
            })
            status, body, error_message = dvid_conn.make_request(
                '/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
                post_json.encode('utf-8'))

    # sync synapses to segmentation
    if dvid_segm is not None:
        syn_sync_json = json.dumps({'sync': dvid_segm})
        dvid_node.custom_request('%s/sync' % dvid_annot,
                                 syn_sync_json.encode('utf-8'),
                                 ConnectionMethod.POST)

    # sync labelsz to synapses
    if dvid_labelsz is not None:
        lsz_sync_json = json.dumps({'sync': dvid_annot})
        dvid_node.custom_request('%s/sync' % dvid_labelsz,
                                 lsz_sync_json.encode('utf-8'),
                                 ConnectionMethod.POST)
Exemplo n.º 39
0
def is_datainstance(dvid_server, uuid, name):
    """Checks if datainstance name exists.

    Args:
        dvid_server (str): location of dvid server
        uuid (str): version id
        name (str): data instance name
    """
    try:
        ns = DVIDNodeService(str(dvid_server), str(uuid))
        info = ns.get_typeinfo(name)
    except DVIDException:
        # returns exception if it does not exist
        return False
    return True
Exemplo n.º 40
0
def is_datainstance(dvid_server, uuid, name):
    """Checks if datainstance name exists.

    Args:
        dvid_server (str): location of dvid server
        uuid (str): version id
        name (str): data instance name
    """
    try:
        ns = DVIDNodeService(str(dvid_server), str(uuid))
        info = ns.get_typeinfo(name)
    except DVIDException:
        # returns exception if it does not exist
        return False
    return True
Exemplo n.º 41
0
    def getImageDataAtTimeFrame(self, Resource, PathInResource, axes,
                                timeframe):
        """
        Loads image data from local resource file in hdf5 format.
        PathInResource provides the internal image path 
        Return numpy array of image data at timeframe.
        """
        node_service = DVIDNodeService(Resource, PathInResource)

        if (self.shape == None):
            self.getImageShape(Resource, PathInResource)

        raw_frame = node_service.get_gray3D(self._getRawImageName(timeframe),
                                            tuple(self.shape), (0, 0, 0))
        return raw_frame
Exemplo n.º 42
0
def test_encode_labelarray_volume():
    vol = np.random.randint(1000, 2000, size=(128, 128, 128), dtype=np.uint64)
    vol[:64, :64, :64] = 0
    vol[-64:, -64:, -64:] = 0

    encoded = encode_labelarray_volume((512, 1024, 2048), vol, gzip_level=0)
    encoded2 = encode_labelarray_volume((512, 1024, 2048), vol, gzip_level=9)
    assert len(encoded) > len(encoded2)

    inflated = DVIDNodeService.inflate_labelarray_blocks3D_from_raw(
        encoded, (128, 128, 128), (512, 1024, 2048))
    assert (inflated == vol).all()

    inflated = DVIDNodeService.inflate_labelarray_blocks3D_from_raw(
        encoded2, (128, 128, 128), (512, 1024, 2048))
    assert (inflated == vol).all()
Exemplo n.º 43
0
def evaluate_pr(tbars_pd, tbars_gt,
                dist_thresh=27, conf_thresholds=None,
                segm_dvid=None):

    tbars_pd = tbars_pd.get_tbars()
    tbars_gt = tbars_gt.get_tbars(1.)

    if conf_thresholds is None:
        cc = np.unique(tbars_pd.conf)
        conf_thresholds = (cc[1:]+cc[:-1])/2

    if segm_dvid is not None:
        segm_dvid_node = DVIDNodeService(
            segm_dvid[0], segm_dvid[1],
            'flyem_syn_eval', 'flyem_syn_eval')
        segm_dvid_segm = segm_dvid[2]

        lbls_pd = get_labels(segm_dvid_node, segm_dvid_segm, tbars_pd)
        lbls_gt = get_labels(segm_dvid_node, segm_dvid_segm, tbars_gt)
    else:
        lbls_pd = None
        lbls_gt = None

    result = obj_pr_curve(tbars_pd, tbars_gt,
                          dist_thresh, conf_thresholds,
                          lbls_pd, lbls_gt)
    return result
Exemplo n.º 44
0
    def test_roi_partition(self):
        node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
        node_service.create_roi("test_roi_partition")
        
        blocks = [(0,0,0),(1,1,1),(2,2,2),(3,3,3)]
        node_service.post_roi("test_roi_partition", blocks)
        substacks, packing_factor = node_service.get_roi_partition("test_roi_partition", 4)
        
        self.assertEqual(substacks, [SubstackXYZ(0,0,0,4*32)])        
        self.assertEqual( packing_factor, float(len(blocks))/(len(substacks) * 4**3) )

        blocks += [(4,0,0)]
        node_service.post_roi("test_roi_partition", blocks)
        substacks, packing_factor = node_service.get_roi_partition("test_roi_partition", 4)
 
        self.assertEqual(substacks, [SubstackXYZ(0,0,0,4*32), SubstackXYZ(128,0,0,4*32)])
Exemplo n.º 45
0
def fri_get_image_generator(get_image_args, qq):
    if not get_image_args:
        return

    dvid_server = get_image_args[0][1]
    dvid_uuid = get_image_args[0][2]
    instance_name = get_image_args[0][7]

    if dvid_server[:5] == 'gs://':  # DICED google bucket
        using_diced = True
        if dvid_server.find(',') >= 0:
            store = DicedStore(*dvid_server.split(','))
        else:
            store = DicedStore(dvid_server)

        repo = store.open_repo(uuid=dvid_uuid)
        dvid_node = repo.get_array(instance_name)
    else:
        using_diced = False
        dvid_node = DVIDNodeService(dvid_server, dvid_uuid, 'fpl', 'fpl')
    for gg in get_image_args:
        while qq.qsize() >= 2:
            time.sleep(10)
        qq.put(fri_get_image(gg, dvid_node, using_diced, instance_name))

    if using_diced:
        store._shutdown_store()
Exemplo n.º 46
0
    def getLabelImageForFrame(self, Resource, PathInResource, timeframe):
        """
        Loads label image data from local resource file in hdf5 format.
        PathInResource provides the internal image path 
        Return numpy array of image data at timeframe.
        """

        if (self.shape == None):
            self.getImageShape(Resource, PathInResource)

        node_service = DVIDNodeService(Resource, PathInResource)
        seg_frame = np.array(
            node_service.get_labels3D(self._getSegmentationName(timeframe),
                                      tuple(self.shape),
                                      (0, 0, 0))).astype(np.uint32)
        return seg_frame
Exemplo n.º 47
0
    def open_repo(self, name=None, uuid=None):
        """Open repository of the specified name (or by unique identifier).

        If only the name is specified, the root node is taken by default.

        Args:
            name (str): name of repository to open
            uuid (str): unique identifier for repository
        
        Returns:
            A repository object if it exists.

        Exception:
            Raises a DicedException if repo does not exist.
        """

        if uuid is not None:
            try:
                ns = DVIDNodeService(self._server, uuid)
                return DicedRepo(self._server, uuid, self)
            except DVIDException:
                # repo does not exist
                raise DicedException("uuid does not exist")
        elif name is not None:
            repoid = self.get_repouuid(name)
            return DicedRepo(self._server, repoid, self)
Exemplo n.º 48
0
def download_to_h5( hostname, uuid, instance, roi, output_filepath, dset_name=None, compression='lzf', overlap_px=0):
    """
    """
    ns = DVIDNodeService(hostname, uuid)
    va = VoxelsAccessor(hostname, uuid, instance, throttle=True)
    
    dset_name = dset_name or instance

    assert roi, "Must provide a ROI"
    logger.info("Downloading {hostname}/api/node/{uuid}/{instance}?roi={roi} to {output_filepath}/{dset_name}".format(**locals()))

    substacks, _packing_factor = ns.get_roi_partition(roi, SUBSTACK_SIZE / DVID_BLOCK_SIZE)

    # Substack tuples are (size, z, y, x)
    substacks_zyx = np.array(substacks)[:, 1:]
    
    # If the user specified an 'overlap', we add it to all substacks.
    # Technically, this isn't very efficient, because a lot of overlapping
    # pixels on the interior of the ROI will be fetched twice.
    substacks_zyx[:,0] -= overlap_px
    substacks_zyx[:,1] += overlap_px

    roi_bb = ( np.min(substacks_zyx, axis=0),
               np.max(substacks_zyx, axis=0)+SUBSTACK_SIZE )
    
    with h5py.File(output_filepath, 'a') as output_file:
        try:
            del output_file[dset_name]
        except KeyError:
            pass
        
        dset = output_file.create_dataset( dset_name, shape=roi_bb[1], dtype=va.dtype, chunks=True, compression=compression )
    
        for i, substack_zyx in enumerate(substacks_zyx):
            logger.info("Substack {}/{} {}: Downloading...".format( i, len(substacks_zyx), list(substack_zyx) ))
            
            # Append a singleton channel axis
            substack_bb = np.array(( tuple(substack_zyx) + (0,),
                                     tuple(substack_zyx + SUBSTACK_SIZE) + (1,) ))
            
            # Includes singleton channel
            substack_data = va.get_ndarray(*substack_bb)

            logger.info("Substack {}/{} {}: Writing...".format( i, len(substacks_zyx), list(substack_zyx) ))
            dset[bb_to_slicing(*substack_bb[:,:-1])] = substack_data[...,0]

    logger.info("DONE Downloading {hostname}/api/node/{uuid}/{instance}?roi={roi} to {output_filepath}/{dset_name}".format(**locals()))
Exemplo n.º 49
0
    def test_keyvalue(self):
        node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
        node_service.create_keyvalue("test_keyvalue")
        node_service.put("test_keyvalue", "kkkk", "vvvv")
        readback_value = node_service.get("test_keyvalue", "kkkk")
        self.assertEqual(readback_value, "vvvv")
 
        with self.assertRaises(RuntimeError):
            node_service.put("test_keyvalue", "kkkk", 123) # 123 is not a buffer.
Exemplo n.º 50
0
    def ingest_data(self):
        """
        Ideally, this would be run within the __init__ function,
        but operators should never raise non-fatal exceptions within Operator.__init__()
        (See OperatorMetaClass.__call__)
        This serves as an alternative init function, from which we are allowed to raise exceptions.
        """
        try:
            node_service = DVIDNodeService(self._hostname, self._uuid)
            roi_blocks_zyx = numpy.array( node_service.get_roi(self._roi_name) )
        except DVIDException as ex:
            if ex.status == httplib.NOT_FOUND:
                raise OpDvidRoi.DatasetReadError("DVIDException: " + ex.message)
            raise
        except ErrMsg as ex:
            raise OpDvidRoi.DatasetReadError("ErrMsg: " + str(ex))

        # Store a dense array of the block flags
        # Each voxel of this dataset represents a dvid block, so this volume
        # is 32x smaller than the full-res data in every dimension.
        # TODO: For now, we only respect the high-side of the bounding box, and force the low-side to 0,0,0.
        shape = tuple(1 + numpy.max( roi_blocks_zyx, axis=0 ))
        slice_shape = (1,) + shape[1:]
        self._mem_file = h5py.File(str(id(self)), driver='core', backing_store=False, mode='w')
        dset = self._mem_file.create_dataset('data',
                                             shape=shape,
                                             dtype=numpy.uint8,
                                             chunks=slice_shape,
                                             compression='lzf')
        
        # Allocate temporary array slice for writing into the dataset
        z_slice = numpy.ndarray( slice_shape, numpy.uint8 )

        # Group the block indexes by z-coordinate
        # FIXME: This might get slow for lots of blocks because of all the calls to this lambda.
        #        Consider upgrading to cytools.groupby instead
        group_iter = groupby(roi_blocks_zyx, lambda block_index: block_index[0])
        
        # Set a pixel for each block index, one z-slice at a time
        for z, block_indexes_iter in group_iter:
            block_indexes = numpy.array(list(block_indexes_iter))
            block_indexes = block_indexes[:,1:] # drop z index
            z_slice[:] = 0
            z_slice[0][ tuple(block_indexes.transpose()) ] = 1
            dset[z,:,:] = z_slice[0,...]
        self._dset = dset
Exemplo n.º 51
0
    def _update_subvol_widget(self, node_uuid, dataname, typename):
        """
        Update the subvolume widget with the min/max extents of the given node and dataname.
        Note: The node and dataname do not necessarily have to match the currently 
              selected node and dataname.
              This enables the right-click behavior, which can be used to  
              limit your data volume to the size of a different data volume.
        """
        error_msg = None
        try:
            if typename == "roi":
                node_service = DVIDNodeService(self._hostname, str(node_uuid))
                roi_blocks_xyz = numpy.array(node_service.get_roi(str(dataname)))
                maxindex = tuple(DVID_BLOCK_WIDTH * (1 + numpy.max(roi_blocks_xyz, axis=0)))
                minindex = (0, 0, 0)  # Rois are always 3D
                axiskeys = "xyz"
                # If the current selection is a dataset, then include a channel dimension
                if self.get_selection().typename != "roi":
                    axiskeys = "cxyz"
                    minindex = (0,) + minindex
                    maxindex = (1,) + maxindex  # FIXME: This assumes that the selected data has only 1 channel...
            else:
                # Query the server
                raw_metadata = VoxelsAccessor.get_metadata(self._hostname, node_uuid, dataname)
                voxels_metadata = VoxelsMetadata(raw_metadata)
                maxindex = voxels_metadata.shape
                minindex = voxels_metadata.minindex
                axiskeys = voxels_metadata.axiskeys
                # If the current selection is a roi, then remove the channel dimension
                if self.get_selection().typename == "roi":
                    axiskeys = "xyz"
                    minindex = minindex[1:]
                    maxindex = maxindex[1:]
        except (DVIDException, ErrMsg) as ex:
            error_msg = str(ex)
            log_exception(logger)
        else:
            self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True)

        if error_msg:
            self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False)
            QMessageBox.critical(self, "DVID Error", error_msg)
            self._subvol_widget.initWithExtents("", (), (), ())
            return

        self._subvol_widget.initWithExtents(axiskeys, maxindex, minindex, maxindex)
Exemplo n.º 52
0
    def test_sparselabelmask(self):
        node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
                                       "*****@*****.**", "test_sparselabelmask")
        node_service.create_labelarray("test_sparselabelmask", 64)

        # Diagonal Stripes
        data = numpy.indices(
            (128, 128, 128),
            dtype=numpy.uint16).sum(axis=0).astype(numpy.uint64)
        data[:] //= 5

        data_downsampled = data[::2, ::2, ::2].copy(
            'C')  # cheap downsampling...

        TEST_LABEL = data[
            64, 64,
            64]  # This is a big diagonal that passes through all but the first block.
        assert TEST_LABEL in data
        assert data.flags['C_CONTIGUOUS']

        # Load two scales
        node_service.put_labelblocks3D("test_sparselabelmask", data, (0, 0, 0))
        node_service.put_labelblocks3D("test_sparselabelmask",
                                       data_downsampled, (0, 0, 0),
                                       scale=1)

        # Retrieve a body at full scale
        block_starts, block_masks = node_service.get_sparselabelmask(
            TEST_LABEL, "test_sparselabelmask", 0)
        assert len(block_starts) == len(block_masks) == 7
        assert block_masks[0].dtype == numpy.bool

        block_stops = block_starts + 64

        for start, stop, mask in zip(block_starts, block_stops, block_masks):
            expected_mask = (data[bb_to_slicing(start, stop)] == TEST_LABEL)
            assert (mask == expected_mask).all()

        # Retrieve that body at half scale
        block_starts, block_masks = node_service.get_sparselabelmask(
            TEST_LABEL, "test_sparselabelmask", 1)
        assert len(block_starts) == len(block_masks) == 1

        expected_mask = (data_downsampled == TEST_LABEL)
        assert (block_starts[0] == (0, 0, 0)).all()
        assert (block_masks[0] == expected_mask).all()
Exemplo n.º 53
0
    def test_labels_3d(self):
        node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid, "*****@*****.**", "test_label_app")
        node_service.create_labelblk("test_labels_3d")
        lblksize = node_service.get_blocksize("test_labels_3d")

        self.assertEqual(32, lblksize)
        
        
        node_service.create_labelblk("test_labels_3d64", "", 64)
        lblksize64 = node_service.get_blocksize("test_labels_3d64")
        self.assertEqual(64, lblksize64)
       
        data = numpy.random.randint(0, 2**63-1, (128,128,128)).astype(numpy.uint64)
        assert data.flags['C_CONTIGUOUS']
        node_service.put_labels3D( "test_labels_3d", data, (0,0,0) )
        retrieved_data = node_service.get_labels3D( "test_labels_3d", (30,31,32), (20,20,20) )
        self.assertTrue( (retrieved_data == data[20:50, 20:51, 20:52]).all() )
Exemplo n.º 54
0
    def parallelize_roi(self, roi, chunk_size, border=0, find_neighbors=False):
        """Creates an RDD from subvolumes found in an ROI.

        This is analogous to the Spark parallelize function.
        It currently defines the number of partitions as the 
        number of subvolumes.

        TODO: implement general partitioner given other
        input such as bounding box coordinates.
        
        Args:
            roi (str): name of DVID ROI at current server and uuid
            chunk_size (int): the desired dimension of the subvolume
            border (int): size of the border surrounding the subvolume
            find_neighbors (bool): whether to identify neighbors

        Returns:
            RDD as [(subvolume id, subvolume)] and # of subvolumes

        """

        # function will export and should include dependencies
        subvolumes = []  # x,y,z,x2,y2,z2

        from libdvid import DVIDNodeService, SubstackXYZ

        # extract roi for a given chunk size
        node_service = DVIDNodeService(str(self.dvid_server), str(self.uuid))
        substacks, packing_factor = node_service.get_roi_partition(str(roi), chunk_size / self.BLK_SIZE)

        # create roi array giving unique substack ids
        for substack_id, substack in enumerate(substacks):
            # use substack id as key
            subvolumes.append((substack_id, Subvolume(substack_id, substack, chunk_size, border)))

        # grab all neighbors for each substack
        if find_neighbors:
            # inefficient search for all boundaries
            for i in range(0, len(subvolumes) - 1):
                for j in range(i + 1, len(subvolumes)):
                    subvolumes[i][1].recordborder(subvolumes[j][1])

        # Potential TODO: custom partitioner for grouping close regions
        return self.sc.parallelize(subvolumes, len(subvolumes))
Exemplo n.º 55
0
    def test_roi_partition(self):
        node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
        node_service.create_roi("test_roi_partition")
        
        blocks = [(0,0,0),(1,1,1),(2,2,2),(3,3,3)]
        node_service.post_roi("test_roi_partition", blocks)
        substacks, packing_factor = node_service.get_roi_partition("test_roi_partition", 4)
        
        self.assertEqual(substacks, [SubstackXYZ(0,0,0,4*32)])        
        self.assertEqual( packing_factor, float(len(blocks))/(len(substacks) * 4**3) )

        blocks += [(4,0,0)]
        node_service.post_roi("test_roi_partition", blocks)
        substacks, packing_factor = node_service.get_roi_partition("test_roi_partition", 4)
 
        self.assertEqual(substacks, [SubstackXYZ(0,0,0,4*32), SubstackXYZ(128,0,0,4*32)])
Exemplo n.º 56
0
def set_sync(dvid_server, uuid, srcname, destname):
    """Sets a sync on srcname to point to destname.
    
    Note: only a limited number of syncs are possible.
    libdvid-cpp will throw an error if a sync is not possible.

    Args:
        dvid_server (str): location of dvid server
        uuid (str): version id
        srcname (str): data instance with new sync
        destname (str): data instance pointed to by new sync 
    
    Raises:
        DVIDExceptions are not caught in this function and will be
        transferred to the caller.
    """

    ns = DVIDNodeService(str(dvid_server), str(uuid))
    data = {"sync": destname}
    ns.custom_request(srcname + "/sync", json.dumps(data).encode('utf-8'), ConnectionMethod.POST)
Exemplo n.º 57
0
    def setUpClass(cls):
        """
        Override.  Called by nosetests.
        """
        cls.uuid = get_testrepo_root_uuid()
        cls.roi_name = "OpDvidRoi_test_roi"
        node_service = DVIDNodeService(TEST_DVID_SERVER, cls.uuid)
        node_service.create_roi(cls.roi_name)

        # Create an upside-down L-shaped roi, something like this:
        #
        # 1 1 1 1
        # 1 1 1 1
        # 1 1 0 0
        # 1 1 0 0
        cls.expected_data = numpy.ones((64, 128, 128), dtype=numpy.uint8, order="C")
        cls.expected_data[:, 64:, 64:] = 0
        block_values = cls.expected_data[::32, ::32, ::32]
        coords = numpy.transpose(numpy.nonzero(block_values))
        node_service.post_roi(cls.roi_name, coords)
Exemplo n.º 58
0
    def __init__(self, dvidserver, uuid, dataname):
        """Initialization.

        Args:
            dvidserver (string): address for dvid server
            uuid (string): uuid for DVID instance 
            dataname (string): name of data instance
        """

        self.server = dvidserver
        self.uuid = uuid
        self.name = dataname
        
        # check DVID existence and get meta
        try:  
            node_service = DVIDNodeService(str(dvidserver), str(uuid))
            self.info = node_service.get_typeinfo(dataname)
        except DVIDException:
            raise ValueError("Instance not available")        
        self.datatype = str(self.info["Base"]["TypeName"])
Exemplo n.º 59
0
def copy_roi( src_info, dest_info ):
    src_service = DVIDNodeService(src_info.server, src_info.uuid)
    dest_service = DVIDNodeService(dest_info.server, dest_info.uuid)

    # If necessary, create the ROI on the destination server
    try:
        info = dest_service.get_typeinfo(dest_info.name)
    except DVIDException:
        dest_service.create_roi(dest_info.name)
    
    
    roi_blocks = src_service.get_roi(src_info.name)
    dest_service.post_roi(dest_info.name, roi_blocks)