Esempio n. 1
0
    def test_special_z_translation(self):
        """
        This tests the special 
        """
        tiled_volume = TiledVolume(
            self.data_setup.SPECIAL_Z_VOLUME_DESCRIPTION_FILE)
        tiled_volume.TEST_MODE = True
        reference_roi = numpy.array([(20, 150, 100), (40, 550, 550)])
        result_out = numpy.zeros(reference_roi[1] - reference_roi[0],
                                 dtype=tiled_volume.description.dtype)

        roi_translated = reference_roi - [11, 0, 0]

        tiled_volume.read(roi_translated, result_out)

        ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH)
        with h5py.File(ref_path_comp.externalPath, 'r') as f:
            ref_data = f[ref_path_comp.internalPath][:]

        expected = ref_data[roiToSlice(*reference_roi)]

        #numpy.save('/tmp/expected.npy', expected)
        #numpy.save('/tmp/result_out.npy', result_out)

        assert (expected == result_out).all()
Esempio n. 2
0
    def setupOutputs(self):
        if self.tiled_volume:
            self.tiled_volume.close()

        # Create a TiledVolume object to read the description file and do the downloads.
        self.tiled_volume = TiledVolume(self.DescriptionFilePath.value)

        self.Output.meta.shape = tuple(self.tiled_volume.output_shape)
        self.Output.meta.dtype = self.tiled_volume.description.dtype
        self.Output.meta.axistags = vigra.defaultAxistags(
            self.tiled_volume.description.output_axes)
        self.Output.meta.prefer_2d = True
        self.Output.meta.nickname = self.tiled_volume.description.name
Esempio n. 3
0
    def testBasic(self):
        tiled_volume = TiledVolume( self.data_setup.VOLUME_DESCRIPTION_FILE )
        tiled_volume.TEST_MODE = True
        roi = numpy.array( [(10, 150, 100), (30, 550, 550)] )
        result_out = numpy.zeros( roi[1] - roi[0], dtype=tiled_volume.description.dtype )
        tiled_volume.read( roi, result_out )
         
        ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH)
        with h5py.File(ref_path_comp.externalPath, 'r') as f:
            ref_data = f[ref_path_comp.internalPath][:]
 
        expected = ref_data[roiToSlice(*roi)]
         
        #numpy.save('/tmp/expected.npy', expected)
        #numpy.save('/tmp/result_out.npy', result_out)
 
        assert (expected == result_out).all()
Esempio n. 4
0
    def testBasic(self):
        tiled_volume = TiledVolume( self.data_setup.LOCAL_VOLUME_DESCRIPTION_FILE )
        tiled_volume.TEST_MODE = True
        roi = numpy.array( [(10, 150, 100), (30, 550, 550)] )
        result_out = numpy.zeros( roi[1] - roi[0], dtype=tiled_volume.description.dtype )
        tiled_volume.read( roi, result_out )
         
        ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH)
        with h5py.File(ref_path_comp.externalPath, 'r') as f:
            ref_data = f[ref_path_comp.internalPath][:]
 
        expected = ref_data[roiToSlice(*roi)]
         
        #numpy.save('/tmp/expected.npy', expected)
        #numpy.save('/tmp/result_out.npy', result_out)
 
        assert (expected == result_out).all()
Esempio n. 5
0
 def testMissingTiles(self):
     tiled_volume = TiledVolume( self.data_setup.VOLUME_DESCRIPTION_FILE )
     tiled_volume.TEST_MODE = True
     # The test data should be missing slice 2, and the config doesn't remap the data.
     roi = numpy.array( [(0, 150, 100), (10, 550, 550)] )
     result_out = numpy.zeros( roi[1] - roi[0], dtype=tiled_volume.description.dtype )
     tiled_volume.read( roi, result_out )
         
     ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH)
     with h5py.File(ref_path_comp.externalPath, 'r') as f:
         ref_data = f[ref_path_comp.internalPath][:]
 
     # Slice 2 is missing from the server data
     expected = ref_data[roiToSlice(*roi)]
     expected[2] = 0
         
     #numpy.save('/tmp/expected.npy', expected)
     #numpy.save('/tmp/result_out.npy', result_out)
 
     assert (expected == result_out).all()
Esempio n. 6
0
 def testMissingTiles(self):
     tiled_volume = TiledVolume( self.data_setup.VOLUME_DESCRIPTION_FILE )
     tiled_volume.TEST_MODE = True
     # The test data should be missing slice 2, and the config doesn't remap the data.
     roi = numpy.array( [(0, 150, 100), (10, 550, 550)] )
     result_out = numpy.zeros( roi[1] - roi[0], dtype=tiled_volume.description.dtype )
     tiled_volume.read( roi, result_out )
         
     ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH)
     with h5py.File(ref_path_comp.externalPath, 'r') as f:
         ref_data = f[ref_path_comp.internalPath][:]
 
     # Slice 2 is missing from the server data
     expected = ref_data[roiToSlice(*roi)]
     expected[2] = 0
         
     #numpy.save('/tmp/expected.npy', expected)
     #numpy.save('/tmp/result_out.npy', result_out)
 
     assert (expected == result_out).all()
Esempio n. 7
0
    def setupOutputs(self):
        if self.tiled_volume:
            self.tiled_volume.close()

        # Create a TiledVolume object to read the description file and do the downloads.
        self.tiled_volume = TiledVolume(self.DescriptionFilePath.value)

        self.Output.meta.shape = tuple(self.tiled_volume.output_shape)
        self.Output.meta.dtype = self.tiled_volume.description.dtype
        self.Output.meta.axistags = vigra.defaultAxistags(self.tiled_volume.description.output_axes)
        self.Output.meta.prefer_2d = True
        self.Output.meta.nickname = self.tiled_volume.description.name
Esempio n. 8
0
class OpTiledVolumeReader(Operator):
    """
    An operator to retrieve volumes from a remote server that provides volumes via image tiles.
    The operator requires a LOCAL json config file that describes the remote dataset and interface.
    (See tiledVolume.py)
    """

    DescriptionFilePath = InputSlot(stype="filestring")
    Output = OutputSlot()

    def __init__(self, *args, **kwargs):
        super(OpTiledVolumeReader, self).__init__(*args, **kwargs)
        self.tiled_volume = None

    def setupOutputs(self):
        if self.tiled_volume:
            self.tiled_volume.close()

        # Create a TiledVolume object to read the description file and do the downloads.
        self.tiled_volume = TiledVolume(self.DescriptionFilePath.value)

        self.Output.meta.shape = tuple(self.tiled_volume.output_shape)
        self.Output.meta.dtype = self.tiled_volume.description.dtype
        self.Output.meta.axistags = vigra.defaultAxistags(self.tiled_volume.description.output_axes)
        self.Output.meta.prefer_2d = True
        self.Output.meta.nickname = self.tiled_volume.description.name

    def execute(self, slot, subindex, roi, result):
        self.tiled_volume.read((roi.start, roi.stop), result)
        return result

    def propagateDirty(self, slot, subindex, roi):
        assert slot == self.DescriptionFilePath, "Unknown input slot."
        self.Output.setDirty(slice(None))

    def cleanUp(self):
        if self.tiled_volume:
            self.tiled_volume.close()
        super(OpTiledVolumeReader, self).cleanUp()
Esempio n. 9
0
class OpTiledVolumeReader(Operator):
    """
    An operator to retrieve volumes from a remote server that provides volumes via image tiles.
    The operator requires a LOCAL json config file that describes the remote dataset and interface.
    (See tiledVolume.py)
    """
    DescriptionFilePath = InputSlot(stype='filestring')
    Output = OutputSlot()

    def __init__(self, *args, **kwargs):
        super(OpTiledVolumeReader, self).__init__(*args, **kwargs)
        self.tiled_volume = None

    def setupOutputs(self):
        if self.tiled_volume:
            self.tiled_volume.close()

        # Create a TiledVolume object to read the description file and do the downloads.
        self.tiled_volume = TiledVolume(self.DescriptionFilePath.value)

        self.Output.meta.shape = tuple(self.tiled_volume.output_shape)
        self.Output.meta.dtype = self.tiled_volume.description.dtype
        self.Output.meta.axistags = vigra.defaultAxistags(
            self.tiled_volume.description.output_axes)
        self.Output.meta.prefer_2d = True
        self.Output.meta.nickname = self.tiled_volume.description.name

    def execute(self, slot, subindex, roi, result):
        self.tiled_volume.read((roi.start, roi.stop), result)
        return result

    def propagateDirty(self, slot, subindex, roi):
        assert slot == self.DescriptionFilePath, "Unknown input slot."
        self.Output.setDirty(slice(None))

    def cleanUp(self):
        if self.tiled_volume:
            self.tiled_volume.close()
        super(OpTiledVolumeReader, self).cleanUp()
Esempio n. 10
0
  def testRemappedTiles(self):
      # The config above specifies that slices 45:47 get their data from slice 44, 
      #  and slice 41 is the same as 40
      tiled_volume = TiledVolume( self.data_setup.VOLUME_DESCRIPTION_FILE )
      tiled_volume.TEST_MODE = True
      roi = numpy.array( [(40, 150, 100), (50, 550, 550)] )
      result_out = numpy.zeros( roi[1] - roi[0], dtype=tiled_volume.description.dtype )
      tiled_volume.read( roi, result_out )
         
      ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH)
      with h5py.File(ref_path_comp.externalPath, 'r') as f:
          ref_data = f[ref_path_comp.internalPath][:]
 
      # Slices 5,6,7 are missing from the server data, and 'filled in' with slice 4
      # Similarly, slice 1 is missing and filled in with slice 0.
      expected = ref_data[roiToSlice(*roi)]
      expected[5:8] = expected[4]
      expected[1] = expected[0]
         
      #numpy.save('/tmp/expected.npy', expected)
      #numpy.save('/tmp/result_out.npy', result_out)
 
      assert (expected == result_out).all()
Esempio n. 11
0
  def testRemappedTiles(self):
      # The config above specifies that slices 45:47 get their data from slice 44, 
      #  and slice 41 is the same as 40
      tiled_volume = TiledVolume( self.data_setup.VOLUME_DESCRIPTION_FILE )
      tiled_volume.TEST_MODE = True
      roi = numpy.array( [(40, 150, 100), (50, 550, 550)] )
      result_out = numpy.zeros( roi[1] - roi[0], dtype=tiled_volume.description.dtype )
      tiled_volume.read( roi, result_out )
         
      ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH)
      with h5py.File(ref_path_comp.externalPath, 'r') as f:
          ref_data = f[ref_path_comp.internalPath][:]
 
      # Slices 5,6,7 are missing from the server data, and 'filled in' with slice 4
      # Similarly, slice 1 is missing and filled in with slice 0.
      expected = ref_data[roiToSlice(*roi)]
      expected[5:8] = expected[4]
      expected[1] = expected[0]
         
      #numpy.save('/tmp/expected.npy', expected)
      #numpy.save('/tmp/result_out.npy', result_out)
 
      assert (expected == result_out).all()
Esempio n. 12
0
    def test_special_z_translation(self):
        """
        This tests the special 
        """
        tiled_volume = TiledVolume( self.data_setup.SPECIAL_Z_VOLUME_DESCRIPTION_FILE )
        tiled_volume.TEST_MODE = True
        reference_roi = numpy.array( [(20, 150, 100), (40, 550, 550)] )
        result_out = numpy.zeros( reference_roi[1] - reference_roi[0], dtype=tiled_volume.description.dtype )

        roi_translated = reference_roi - [11,0,0]
        
        tiled_volume.read( roi_translated, result_out )
         
        ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH)
        with h5py.File(ref_path_comp.externalPath, 'r') as f:
            ref_data = f[ref_path_comp.internalPath][:]
 
        expected = ref_data[roiToSlice(*reference_roi)]
         
        #numpy.save('/tmp/expected.npy', expected)
        #numpy.save('/tmp/result_out.npy', result_out)
 
        assert (expected == result_out).all()
Esempio n. 13
0
 def setup(self):
     """
     Generate a directory with all the files needed for this test.
     We use the same temporary directory every time, so we don't 
     waste time regenerating the data if the test has already been run recently.
     
     The directory consists of the following files:
     - reference_volume.h5
     - volume_description.json
     - transposed_volume_description.json
     - [lots of png tiles..]
     """        
     tmp = tempfile.gettempdir()
     self.TILE_DIRECTORY = os.path.join( tmp, 'testTiledVolume_data' )
     logger.debug("Using test directory: {}".format( self.TILE_DIRECTORY ))
     self.REFERENCE_VOL_PATH = os.path.join( self.TILE_DIRECTORY, 'reference_volume.h5/data' )
     ref_vol_path_comp = PathComponents(self.REFERENCE_VOL_PATH)
     self.REFERENCE_VOL_FILE = ref_vol_path_comp.externalPath
     self.VOLUME_DESCRIPTION_FILE = os.path.join( self.TILE_DIRECTORY, 'volume_description.json' )
     self.TRANSPOSED_VOLUME_DESCRIPTION_FILE = os.path.join( self.TILE_DIRECTORY, 'transposed_volume_description.json' )
 
     if not os.path.exists(self.TILE_DIRECTORY):
         print "Creating new tile directory: {}".format( self.TILE_DIRECTORY )
         os.mkdir(self.TILE_DIRECTORY)
 
     if not os.path.exists(self.REFERENCE_VOL_FILE):
         ref_vol = numpy.random.randint(0,255, (100,600,600) ).astype(numpy.uint8)
         with h5py.File(self.REFERENCE_VOL_FILE, 'w') as ref_file:
             ref_file[ref_vol_path_comp.internalPath] = ref_vol
     else:
         with h5py.File(self.REFERENCE_VOL_FILE, 'r') as ref_file:
             ref_vol = ref_file[ref_vol_path_comp.internalPath][:] 
 
     need_rewrite = False
     if not os.path.exists( self.VOLUME_DESCRIPTION_FILE ):
         need_rewrite = True
     else:
         with open(self.VOLUME_DESCRIPTION_FILE, 'r') as f:
             if f.read() != volume_description_text:
                 need_rewrite = True
     
     if need_rewrite:
         with open(self.VOLUME_DESCRIPTION_FILE, 'w') as f:
             f.write(volume_description_text)
 
         # Read the volume description as a JsonConfig Namespace
         volume_description = TiledVolume.readDescription(self.VOLUME_DESCRIPTION_FILE)
 
         # Write out a copy of the description, but with custom output axes
         config_helper = JsonConfigParser( TiledVolume.DescriptionFields )
         transposed_description = copy.copy(volume_description)
         transposed_description.output_axes = "xyz"
         config_helper.writeConfigFile(self.TRANSPOSED_VOLUME_DESCRIPTION_FILE, transposed_description)
 
         # Remove all old image tiles in the tile directory
         files = os.listdir(self.TILE_DIRECTORY)
         for name in files:
             if os.path.splitext(name)[1] == '.' + volume_description.format:
                 os.remove( os.path.join(self.TILE_DIRECTORY, name) )
 
         # Write the new tiles
         export_to_tiles( ref_vol, volume_description.tile_shape_2d_yx[0], self.TILE_DIRECTORY, print_progress=False )    
 
         # To support testMissingTiles (below), remove slice 2
         files = os.listdir(self.TILE_DIRECTORY)
         for name in files:
             if name.startswith("tile_z00002"):
                 p = os.path.join(self.TILE_DIRECTORY, name)
                 print "removing:", p
                 os.remove( p )
             
 
     # lastly, start the server
     self._start_server()
Esempio n. 14
0
    def setup(self):
        """
        Generate a directory with all the files needed for this test.
        We use the same temporary directory every time, so we don't 
        waste time regenerating the data if the test has already been run recently.
        
        The directory consists of the following files:
        - reference_volume.h5
        - volume_description.json
        - transposed_volume_description.json
        - [lots of png tiles..]
        """
        tmp = tempfile.gettempdir()
        self.TILE_DIRECTORY = os.path.join(tmp, 'testTiledVolume_data')
        logger.debug("Using test directory: {}".format(self.TILE_DIRECTORY))
        self.REFERENCE_VOL_PATH = os.path.join(self.TILE_DIRECTORY,
                                               'reference_volume.h5/data')
        ref_vol_path_comp = PathComponents(self.REFERENCE_VOL_PATH)
        self.REFERENCE_VOL_FILE = ref_vol_path_comp.externalPath
        self.VOLUME_DESCRIPTION_FILE = os.path.join(self.TILE_DIRECTORY,
                                                    'volume_description.json')
        self.TRANSPOSED_VOLUME_DESCRIPTION_FILE = os.path.join(
            self.TILE_DIRECTORY, 'transposed_volume_description.json')

        if not os.path.exists(self.TILE_DIRECTORY):
            print "Creating new tile directory: {}".format(self.TILE_DIRECTORY)
            os.mkdir(self.TILE_DIRECTORY)

        if not os.path.exists(self.REFERENCE_VOL_FILE):
            ref_vol = numpy.random.randint(0, 255,
                                           (100, 600, 600)).astype(numpy.uint8)
            with h5py.File(self.REFERENCE_VOL_FILE, 'w') as ref_file:
                ref_file[ref_vol_path_comp.internalPath] = ref_vol
        else:
            with h5py.File(self.REFERENCE_VOL_FILE, 'r') as ref_file:
                ref_vol = ref_file[ref_vol_path_comp.internalPath][:]

        need_rewrite = False
        if not os.path.exists(self.VOLUME_DESCRIPTION_FILE):
            need_rewrite = True
        else:
            with open(self.VOLUME_DESCRIPTION_FILE, 'r') as f:
                if f.read() != volume_description_text:
                    need_rewrite = True

        if need_rewrite:
            with open(self.VOLUME_DESCRIPTION_FILE, 'w') as f:
                f.write(volume_description_text)

            # Read the volume description as a JsonConfig Namespace
            volume_description = TiledVolume.readDescription(
                self.VOLUME_DESCRIPTION_FILE)

            # Write out a copy of the description, but with custom output axes
            config_helper = JsonConfigParser(TiledVolume.DescriptionFields)
            transposed_description = copy.copy(volume_description)
            transposed_description.output_axes = "xyz"
            config_helper.writeConfigFile(
                self.TRANSPOSED_VOLUME_DESCRIPTION_FILE,
                transposed_description)

            # Remove all old image tiles in the tile directory
            files = os.listdir(self.TILE_DIRECTORY)
            for name in files:
                if os.path.splitext(
                        name)[1] == '.' + volume_description.format:
                    os.remove(os.path.join(self.TILE_DIRECTORY, name))

            # Write the new tiles
            export_to_tiles(ref_vol,
                            volume_description.tile_shape_2d_yx[0],
                            self.TILE_DIRECTORY,
                            print_progress=False)

            # To support testMissingTiles (below), remove slice 2
            files = os.listdir(self.TILE_DIRECTORY)
            for name in files:
                if name.startswith("tile_z00002"):
                    p = os.path.join(self.TILE_DIRECTORY, name)
                    print "removing:", p
                    os.remove(p)

        # lastly, start the server
        self._start_server()