Ejemplo n.º 1
0
    def test_get_cubes(self):
        """Test adding cubes to the cache"""
        resolution = 1
        rkv = RedisKVIO(self.config_data)

        # Clean up data
        self.cache_client.flushdb()

        data1 = np.random.randint(50, size=[10, 15, 5])
        data2 = np.random.randint(50, size=[10, 15, 5])
        data3 = np.random.randint(50, size=[10, 15, 5])
        data_packed1 = blosc.pack_array(data1)
        data_packed2 = blosc.pack_array(data2)
        data_packed3 = blosc.pack_array(data3)
        data = [data_packed1, data_packed2, data_packed3]

        # Add items
        morton_id = [112, 125, 516]
        keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)
        rkv.put_cubes(keys, data)

        # Get cube
        cubes = rkv.get_cubes(keys)
        assert len(cubes) == 3

        for m, c, d in zip(morton_id, cubes, data):
            assert c[0] == m
            assert c[1] == 0
            data_retrieved = blosc.unpack_array(c[2])
            np.testing.assert_array_equal(data_retrieved, blosc.unpack_array(d))
Ejemplo n.º 2
0
    def test_put_cubes(self):
        """Test adding cubes to the cache"""
        resolution = 1
        rkv = RedisKVIO(self.config_data)

        # Clean up data
        self.cache_client.flushdb()

        data1 = np.random.randint(50, size=[10, 15, 5])
        data2 = np.random.randint(50, size=[10, 15, 5])
        data3 = np.random.randint(50, size=[10, 15, 5])
        data_packed1 = blosc.pack_array(data1)
        data_packed2 = blosc.pack_array(data2)
        data_packed3 = blosc.pack_array(data3)
        data = [data_packed1, data_packed2, data_packed3]

        # Make sure there are no cuboids in the cache
        keys = self.cache_client.keys('CACHED-CUBOID&{}&{}*'.format(self.resource.get_lookup_key(), resolution))
        assert not keys

        # Add items
        keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], [123, 124, 126])
        rkv.put_cubes(keys, data)

        db_keys = self.cache_client.keys('CACHED-CUBOID*')
        db_keys = [x.decode() for x in db_keys]
        assert len(set(keys)) == 3
        for k, d in zip(keys, db_keys):
            assert k in db_keys
Ejemplo n.º 3
0
    def test_get_cubes(self):
        """Test adding cubes to the cache"""
        resolution = 1
        rkv = RedisKVIO(self.config_data)

        # Clean up data
        self.cache_client.flushdb()

        data1 = np.random.randint(50, size=[10, 15, 5])
        data2 = np.random.randint(50, size=[10, 15, 5])
        data3 = np.random.randint(50, size=[10, 15, 5])
        data_packed1 = blosc.pack_array(data1)
        data_packed2 = blosc.pack_array(data2)
        data_packed3 = blosc.pack_array(data3)
        data = [data_packed1, data_packed2, data_packed3]

        # Add items
        morton_id = [112, 125, 516]
        keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
                                               morton_id)
        rkv.put_cubes(keys, data)

        # Get cube
        cubes = rkv.get_cubes(keys)
        assert len(cubes) == 3

        for m, c, d in zip(morton_id, cubes, data):
            assert c[0] == m
            assert c[1] == 0
            data_retrieved = blosc.unpack_array(c[2])
            np.testing.assert_array_equal(data_retrieved,
                                          blosc.unpack_array(d))
Ejemplo n.º 4
0
 def supercube_compatibility(self, super_cube):
   
   super_cube = blosc.unpack_array(super_cube)
   if len(super_cube.shape) == 3:
     return blosc.pack_array(super_cube.reshape((1,) + super_cube.shape))
   else:
     return blosc.pack_array(super_cube)
Ejemplo n.º 5
0
  def updateIndex ( self, ch, entityid, index, resolution ):
    """Updated the database index table with the input index hash table"""

    curindex = self.getIndex(ch, entityid, resolution, True)

    if curindex == []:
        
      if self.NPZ:
        fileobj = cStringIO.StringIO ()
        np.save ( fileobj, index )
        self.kvio.putIndex(ch, entityid, resolution, fileobj.getvalue())
      else:
        self.kvio.putIndex(ch, entityid, resolution, blosc.pack_array(index))

    else :
        
      # Update Index to the union of the currentIndex and the updated index
      newIndex = np.union1d(curindex, index)

      # Update the index in the database
      if self.NPZ:
        fileobj = cStringIO.StringIO ()
        np.save ( fileobj, newIndex )
        self.kvio.putIndex(ch, entityid, resolution, fileobj.getvalue(), True)
      else:
        self.kvio.putIndex(ch, entityid, resolution, blosc.pack_array(newIndex), True)
Ejemplo n.º 6
0
    def __read_img_label_into_zipnp(self,img_label_path_dic,img_label_dic):
        pbar = pb.ProgressBar(widgets=[pb.Percentage(), pb.Bar(), pb.ETA()], maxval=len(img_label_path_dic)).start()
        count = 0
        for fn, img_label_path in img_label_path_dic.items():
            img_label_np_dic = {}
            img_sitk, original_spacing, original_sz = self.__read_and_clean_itk_info(img_label_path['image'])
            resized_img, resize_factor = self.resize_img(img_sitk)
            img_np = sitk.GetArrayFromImage(resized_img)
            img_np = self.normalize_intensity(img_np)
            img_label_np_dic['image'] = blosc.pack_array(img_np.astype(np.float32))

            if self.has_label:
                label_sitk, _, _ = self.__read_and_clean_itk_info(img_label_path['label'])
                resized_label,_ = self.resize_img(label_sitk,is_label=True)
                label_np = sitk.GetArrayFromImage(resized_label)
                label_index = list(np.unique(label_np))
                img_label_np_dic['label'] = blosc.pack_array(label_np.astype(np.int64))
                img_label_np_dic['label_index'] = label_index
            img_after_resize = self.img_after_resize if self.img_after_resize is not None else original_sz
            new_spacing=  original_spacing*(original_sz-1)/(np.array(img_after_resize)-1)
            normalized_spacing = self._normalize_spacing(new_spacing,img_after_resize, silent_mode=True)
            img_label_np_dic['original_sz'] =original_sz
            img_label_np_dic['original_spacing'] = original_spacing
            img_label_np_dic['spacing'] = normalized_spacing
            img_label_np_dic['img_sz'] = list(img_np.shape)
            img_label_dic[fn] =img_label_np_dic
            count +=1
            pbar.update(count)
        pbar.finish()
Ejemplo n.º 7
0
    def updateIndex(self, ch, entityid, index, resolution):
        """Updated the database index table with the input index hash table"""

        curindex = self.getIndex(ch, entityid, resolution, True)

        if curindex == []:

            if self.NPZ:
                fileobj = cStringIO.StringIO()
                np.save(fileobj, index)
                self.kvio.putIndex(ch, entityid, resolution,
                                   fileobj.getvalue())
            else:
                self.kvio.putIndex(ch, entityid, resolution,
                                   blosc.pack_array(index))

        else:

            # Update Index to the union of the currentIndex and the updated index
            newIndex = np.union1d(curindex, index)

            # Update the index in the database
            if self.NPZ:
                fileobj = cStringIO.StringIO()
                np.save(fileobj, newIndex)
                self.kvio.putIndex(ch, entityid, resolution,
                                   fileobj.getvalue(), True)
            else:
                self.kvio.putIndex(ch, entityid, resolution,
                                   blosc.pack_array(newIndex), True)
Ejemplo n.º 8
0
    def test_put_cubes(self):
        """Test adding cubes to the cache"""
        resolution = 1
        rkv = RedisKVIO(self.config_data)

        # Clean up data
        self.cache_client.flushdb()

        data1 = np.random.randint(50, size=[10, 15, 5])
        data2 = np.random.randint(50, size=[10, 15, 5])
        data3 = np.random.randint(50, size=[10, 15, 5])
        data_packed1 = blosc.pack_array(data1)
        data_packed2 = blosc.pack_array(data2)
        data_packed3 = blosc.pack_array(data3)
        data = [data_packed1, data_packed2, data_packed3]

        # Make sure there are no cuboids in the cache
        keys = self.cache_client.keys('CACHED-CUBOID&{}&{}*'.format(
            self.resource.get_lookup_key(), resolution))
        assert not keys

        # Add items
        keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
                                               [123, 124, 126])
        rkv.put_cubes(keys, data)

        db_keys = self.cache_client.keys('CACHED-CUBOID*')
        db_keys = [x.decode() for x in db_keys]
        assert len(set(keys)) == 3
        for k, d in zip(keys, db_keys):
            assert k in db_keys
Ejemplo n.º 9
0
    def supercube_compatibility(self, super_cube):

        super_cube = blosc.unpack_array(super_cube)
        if len(super_cube.shape) == 3:
            return blosc.pack_array(
                super_cube.reshape((1, ) + super_cube.shape))
        else:
            return blosc.pack_array(super_cube)
Ejemplo n.º 10
0
    def render(self, data, media_type=None, renderer_context=None):

        if not data["data"].data.flags['C_CONTIGUOUS']:
            data["data"].data = np.ascontiguousarray(data["data"].data, dtype=data["data"].data.dtype)

        # Return data, squeezing time dimension if only a single point
        if data["time_request"]:
            return blosc.pack_array(data["data"].data)
        else:
            return blosc.pack_array(np.squeeze(data["data"].data, axis=(0,)))
Ejemplo n.º 11
0
def send_data(socket, *inputs):
    """Pack the array and send it"""
    if len(inputs) == 1:
        packed_array = blosc.pack_array(inputs[0])
        return socket.send((packed_array, ))

    elif len(inputs) == 4:
        packed_array = blosc.pack_array(inputs[0])

        return socket.send((packed_array, ) + (inputs[1:]))
Ejemplo n.º 12
0
    def render(self, data, media_type=None, renderer_context=None):

        if not data.data.flags['C_CONTIGUOUS']:
            data.data = data.data.copy(order='C')

        # Return data, squeezing time dimension if only a single point
        try:
            return blosc.pack_array(np.squeeze(data.data, axis=(0, )))
        except ValueError:
            # The are more than 1 time points so don't squeeze.
            return blosc.pack_array(data.data)
Ejemplo n.º 13
0
    def stepEnv(self, action=None):
        # If no manual action was specified by the user
        if action is None:
            action = random.randint(0, self.env.action_space.n - 1)
        action = int(action)

        obs, reward, done, info = self.env.step(action)

        self.current_actions.append(action)
        self.current_images.append(self.lastObs['image'])
        self.current_directions.append(self.lastObs['direction'])

        self.showEnv(obs)
        self.lastObs = obs

        if done:
            if reward > 0:  # i.e. we did not lose
                if self.shift < len(self.demos):
                    self.demos[self.shift] = self.current_demo, self.shift
                else:
                    self.demos.append(
                        (self.current_mission,
                         blosc.pack_array(np.array(self.current_images)),
                         self.current_directions, self.current_actions))
                utils.save_demos(self.demos, self.demos_path)
                self.missionBox.append('Demonstrations are saved.')
                utils.synthesize_demos(self.demos)

                self.shift += 1
                self.resetEnv()
            else:
                self.shiftEnv()
Ejemplo n.º 14
0
  def breakCubes(self, timestamp, super_zidx, resolution, super_cube):
    """Breaking the supercuboids into cuboids"""
    
    super_cube = blosc.unpack_array(super_cube)
    # print "breaking supercube shape: {}".format(super_cube.shape)
    # Empty lists for zindx and cube data
    zidx_list = []
    cube_list = []
    
    # SuperCube Size
    [xnumcubes, ynumcubes, znumcubes] = self.db.datasetcfg.supercube_size
    
    # Cube dimensions
    cubedim = self.db.datasetcfg.get_cubedim(resolution)
    [x,y,z] = MortonXYZ(super_zidx)
    # start = map(mul, cubedim, [x,y,z])
    start = map(mul, [x,y,z], self.db.datasetcfg.supercube_size)
    
    for z in range(znumcubes):
      for y in range(ynumcubes):
        for x in range(xnumcubes):
          zidx = XYZMorton(map(add, start, [x,y,z]))

          # Parameters in the cube slab
          index = map(mul, cubedim, [x,y,z])
          end = map(add, index, cubedim)

          cube_data = super_cube[:,index[2]:end[2], index[1]:end[1], index[0]:end[0]]
          zidx_list.append(zidx)
          # print "mini cube:", cube_data.shape
          cube_list.append(blosc.pack_array(cube_data))
    
    return zidx_list, [timestamp]*len(zidx_list), cube_list
Ejemplo n.º 15
0
def postBlosc(p, post_data, time=False, neariso=False, direct=False):
    """Post data using blosc packed numpy array"""

    # Build the url and then create a npz object
    if time:
        url = 'https://{}/sd/{}/{}/blosc/{}/{},{}/{},{}/{},{}/{},{}/'.format(
            SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args)
    elif p.channels is not None:
        url = 'https://{}/sd/{}/{}/blosc/{}/{},{}/{},{}/{},{}/'.format(
            SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args)
    elif p.channels is None:
        url = 'https://{}/sd/{}/blosc/{}/{},{}/{},{}/{},{}/'.format(
            SITE_HOST, p.token, p.resolution, *p.args)

    if neariso:
        url = url + NEARISO

    if direct:
        url = url + DIRECT

    try:
        # Build a post request
        resp = postURL(url, blosc.pack_array(post_data))
        return resp
    except Exception as e:
        return e
Ejemplo n.º 16
0
 def singleThreadTest(self, start_value, size_iterations):
   """Generate the URL for read test"""
   
   min_values = [xmin,ymin,zmin] = map(add, self.info_interface.offset(self.resolution), start_value)
   max_values = map(add, min_values, self.info.cuboid_dimension(self.resolution))
   range_args = [None]*(len(min_values)+len(max_values))
   
   for i in range(0, size_iterations, 1):
     if all([a<b for a,b in zip(max_values, self.info_interface.image_size(self.resolution))]):
       range_args[::2] = min_values
       range_args[1::2] = max_values
       # print "Size", reduce(mul, map(sub, max_values, min_values), 1)*2.0/(1024*1024)
       if self.write_tests:
         data = blosc.pack_array(np.ones( [len(self.channels)]+map(sub, range_args[1::2], range_args[::2])[::-1], dtype=self.datatype) * random.randint(0,255))
         self.data_list.append(data)
         self.fetch_list.append(generateURLBlosc(self.host_name, self.token_name, self.channels, self.resolution, range_args, direct=self.direct))
       else:
         self.fetch_list.append(generateURLBlosc(self.host_name, self.token_name, self.channels, self.resolution, range_args, direct=self.direct))
       # min_values = max_values
       # max_values = map(add, map(sub, min_values, temp_values), min_values)
       min_values[2] = max_values[2]
       max_values[2] = min_values[2] + self.info.cuboid_dimension(self.resolution)[2] 
       max_values[(i+1)%2] = start_value[(i+1)%2] + (max_values[(i+1)%2]-min_values[(i+1)%2])*2
     else:
       break
Ejemplo n.º 17
0
    def _post_cutout_no_chunking_blosc(self, token, channel,
                                       x_start, y_start, z_start,
                                       data, resolution):
        """
        Accepts data in zyx. !!!
        """
        data = numpy.expand_dims(data, axis=0)
        blosc_data = blosc.pack_array(data)

        url = self.url("{}/{}/blosc/{}/{},{}/{},{}/{},{}/".format(
            token, channel,
            resolution,
            x_start, x_start + data.shape[3],
            y_start, y_start + data.shape[2],
            z_start, z_start + data.shape[1]
        ))

        req = requests.post(url, data=blosc_data, headers={
            'Content-Type': 'application/octet-stream'
        })

        if req.status_code is not 200:
            raise RemoteDataUploadError(req.text)
        else:
            return True
Ejemplo n.º 18
0
    def test_delete_cube(self):
        """Test cube delete method"""
        resolution = 1
        rkv = RedisKVIO(self.config_data)

        # Clean up data
        self.cache_client.flushdb()

        data1 = np.random.randint(50, size=[10, 15, 5])
        data_packed1 = blosc.pack_array(data1)
        data = [data_packed1]

        # Add items
        morton_id = [112]
        keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)

        result = rkv.cube_exists(keys[0])
        assert not result

        rkv.put_cubes(keys, data)

        result = rkv.cube_exists(keys[0])
        assert result

        rkv.delete_cube(keys[0])

        result = rkv.cube_exists(keys[0])
        assert not result
Ejemplo n.º 19
0
    def test_is_dirty(self):
        """Test cube delete method"""
        resolution = 1
        rkv = RedisKVIO(self.config_data)

        # Clean up data
        self.cache_client.flushdb()

        data1 = np.random.randint(50, size=[10, 15, 5])
        data_packed1 = blosc.pack_array(data1)
        data = [data_packed1]

        # Add item to cached-cuboid
        morton_id = [112]
        keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)
        rkv.put_cubes(keys, data)

        result = rkv.is_dirty(keys)
        assert not result[0]

        # Fake a write
        write_cuboid_key = "WRITE-CUBOID&{}".format(keys[0].split('&', 1)[1])
        write_cuboid_key, time_sample, morton = write_cuboid_key.rsplit('&', 2)
        rkv.insert_cube_in_write_buffer(write_cuboid_key, time_sample, morton, data[0])

        result = rkv.is_dirty(keys)
        assert result[0]
Ejemplo n.º 20
0
    def test_delete_cube(self):
        """Test cube delete method"""
        resolution = 1
        rkv = RedisKVIO(self.config_data)

        # Clean up data
        self.cache_client.flushdb()

        data1 = np.random.randint(50, size=[10, 15, 5])
        data_packed1 = blosc.pack_array(data1)
        data = [data_packed1]

        # Add items
        morton_id = [112]
        keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
                                               morton_id)

        result = rkv.cube_exists(keys[0])
        assert not result

        rkv.put_cubes(keys, data)

        result = rkv.cube_exists(keys[0])
        assert result

        rkv.delete_cube(keys[0])

        result = rkv.cube_exists(keys[0])
        assert not result
Ejemplo n.º 21
0
    def cutout_create(
        self, resource, resolution, x_range, y_range, z_range, time_range, numpyVolume,
        url_prefix, auth, session, send_opts):
        """Upload a cutout to the Boss data store.

        Args:
            resource (ndio.ndresource.resource.Resource): Resource compatible with cutout operations.
            resolution (int): 0 indicates native resolution.
            x_range (string): x range such as '10:20' which means x>=10 and x<20.
            y_range (string): y range such as '10:20' which means y>=10 and y<20.
            z_range (string): z range such as '10:20' which means z>=10 and z<20.
            time_range (string): None or time range such as 30:40 which means t>=30 and t<40.
            numpyVolume (numpy.array): A 3D or 4D (time) numpy matrix in (time)ZYX order.
            url_prefix (string): Protocol + host such as https://api.theboss.io
            auth (string): Token to send in the request header.
            session (requests.Session): HTTP session to use for request.
            send_opts (dictionary): Additional arguments to pass to session.send().
        """

        compressed = blosc.pack_array(numpyVolume)
        req = self.get_cutout_request(
            resource, 'POST', 'application/blosc-python',
            url_prefix, auth,
            resolution, x_range, y_range, z_range, time_range, compressed)
        prep = session.prepare_request(req)
        resp = session.send(prep, **send_opts)
        
        if resp.status_code == 201:
            return

        msg = ('Create cutout failed on {}, got HTTP response: ({}) - {}'.format(
            resource.name, resp.status_code, resp.text))
        raise HTTPError(msg, request = req, response = resp)
Ejemplo n.º 22
0
    def test_is_dirty(self):
        """Test cube delete method"""
        resolution = 1
        rkv = RedisKVIO(self.config_data)

        # Clean up data
        self.cache_client.flushdb()

        data1 = np.random.randint(50, size=[10, 15, 5])
        data_packed1 = blosc.pack_array(data1)
        data = [data_packed1]

        # Add item to cached-cuboid
        morton_id = [112]
        keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
                                               morton_id)
        rkv.put_cubes(keys, data)

        result = rkv.is_dirty(keys)
        assert not result[0]

        # Fake a write
        write_cuboid_key = "WRITE-CUBOID&{}".format(keys[0].split('&', 1)[1])
        write_cuboid_key, time_sample, morton = write_cuboid_key.rsplit('&', 2)
        rkv.insert_cube_in_write_buffer(write_cuboid_key, time_sample, morton,
                                        data[0])

        result = rkv.is_dirty(keys)
        assert result[0]
Ejemplo n.º 23
0
    def __get_clean_label(self,img_label_dict, img_name_list):
        """

        :param img_label_dict:
        :param img_name_list:
        :return:
        """
        print(" Attention, the annotation for background is assume to be 0 ! ")
        print(" Attention, we are using the union set of the label! ")
        if self.interested_label_list is None:
            interested_label_set = set()
            for i, fname in enumerate(img_name_list):
                label_set = img_label_dict[fname]['label_index']
                if i ==0:
                    interested_label_set = set(label_set)
                else:
                    interested_label_set = interested_label_set.union(label_set)
            interested_label_list = list(interested_label_set)
        else:
            interested_label_list = self.interested_label_list

        #self.standard_label_index = tuple([int(item) for item in interested_label_list])
        for fname in img_name_list:
            label = img_label_dict[fname]['label']
            label = self.__convert_to_standard_label_map(label, interested_label_list)
            label_density = list(np.bincount(label.reshape(-1).astype(np.int32)) / len(label.reshape(-1)))
            img_label_dict[fname]['label'] = blosc.pack_array(label)
            img_label_dict[fname]['label_density']=label_density
            img_label_dict[fname]['label_org_index'] = interested_label_list
            img_label_dict[fname]['label_converted_index'] = list(range(len(interested_label_list)))
        return img_label_dict
Ejemplo n.º 24
0
def postBlosc(p, post_data):
    """Post data using the blosc interface"""

    # Build the url and then create a hdf5 object
    url = 'http://{}/{}/{}/blosc/{}/{},{}/{},{}/{},{}/'.format(
        SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args)
    return (url, blosc.pack_array(post_data))
Ejemplo n.º 25
0
    def test_channel_uint16_wrong_dimensions_numpy(self):
        """ Test posting with the wrong xyz dims using the numpy interface"""

        test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128))
        test_mat = test_mat.astype(np.uint16)
        bb = blosc.pack_array(test_mat)

        # Create request
        factory = APIRequestFactory()
        request = factory.post(
            '/' + version + '/cutout/col1/exp1/channel2/0/0:100/0:128/0:16/',
            bb,
            content_type='application/blosc-python')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request,
                                    collection='col1',
                                    experiment='exp1',
                                    channel='channel2',
                                    resolution='0',
                                    x_range='0:100',
                                    y_range='0:128',
                                    z_range='0:16',
                                    t_range=None)
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
Ejemplo n.º 26
0
    def test_cutout_get_success(self, mock_session):
        resolution = 0
        x_range = '20:40'
        y_range = '50:70'
        z_range = '30:50'
        time_range = '10:25'
        url_prefix = 'https://api.theboss.io'
        auth = 'mytoken'

        fake_prepped_req = PreparedRequest()
        fake_prepped_req.headers = {}
        mock_session.prepare_request.return_value = fake_prepped_req 

        data = numpy.random.randint(0, 3000, (15, 20, 20, 20), numpy.uint16)
        compressed_data = blosc.pack_array(data)
        fake_response = Response()
        fake_response.status_code = 200
        fake_response._content = compressed_data
        mock_session.send.return_value = fake_response
        send_opts = {}

        actual = self.vol.cutout_get(
            self.chan, resolution, x_range, y_range, z_range, time_range,
            url_prefix, auth, mock_session, send_opts)

        numpy.testing.assert_array_equal(data, actual)
Ejemplo n.º 27
0
  def uploadExistingProject(self, channel_name, resolution, start_values, neariso=False):
    """Upload an existing project to S3"""
      
    self.setupNewProject()
    db = SpatialDB(self.proj)
    # checking for channels
    if channel_name is None:
      channel_list = None
    else:
      channel_list = [channel_name]
    
    # iterating over channels in a project
    for ch in self.proj.projectChannels(channel_list):
      
      # creating the channel resource
      self.resource_interface.createChannel(ch.channel_name)
      # ingest 1 or more resolutions based on user input
      if resolution is None:
        start_res = self.proj.datasetcfg.scalinglevels
        stop_res = ch.resolution - 1
      else:
        start_res = resolution
        stop_res = resolution - 1
      
      # iterating over resolution
      for cur_res in range(start_res, stop_res, -1):
        
        # get the source database sizes
        [image_size, time_range] = self.proj.datasetcfg.dataset_dim(cur_res)
        [xcubedim, ycubedim, zcubedim] = cubedim = self.proj.datasetcfg.get_cubedim(cur_res)
        offset = self.proj.datasetcfg.get_offset(cur_res)
        [xsupercubedim, ysupercubedim, zsupercubedim] = supercubedim = self.proj.datasetcfg.get_supercubedim(cur_res)
        # set the limits for iteration on the number of cubes in each dimension
        xlimit = (image_size[0]-1) / (xsupercubedim) + 1
        ylimit = (image_size[1]-1) / (ysupercubedim) + 1
        zlimit = (image_size[2]-1) / (zsupercubedim) + 1
        # [xlimit, ylimit, zlimit] = limit = self.proj.datasetcfg.get_supercube_limit(cur_res)
        [x_start, y_start, z_start] = map(div, start_values, supercubedim)
        for z in range(z_start, zlimit, 1):
          for y in range(y_start, ylimit, 1):
            for x in range(x_start, xlimit, 1):

              try:
                # cutout the data at the current resolution
                data = db.cutout(ch, [x*xsupercubedim, y*ysupercubedim, z*zsupercubedim], [xsupercubedim, ysupercubedim, zsupercubedim], cur_res).data
                # generate the morton index
                morton_index = XYZMorton([x, y, z])

                self.logger.info("[{},{},{}] at res {}".format(x*xsupercubedim, y*ysupercubedim, z*zsupercubedim, cur_res))
                # updating the index
                # self.cuboidindex_db.putItem(ch.channel_name, cur_res, x, y, z, ch.time_range[0])
                # inserting the cube
                self.s3_io.putCube(ch, ch.time_stamp[0], morton_index, cur_res, blosc.pack_array(data), neariso=neariso)
              
              except Exception as e:
                # checkpoint the ingest
                self.logger.error(e)
                self.checkpoint_ingest(ch.channel_name, cur_res, x, y, z, e)
                raise e
Ejemplo n.º 28
0
 def upload(self, file_name, channel_name, resolution, x_index, y_index, z_index, dimensions=[1, 64, 512,512], time_index=0, neariso=False):
   """Upload a 4D supercuboid directly to dynamo and s3"""
   cuboid_data = np.fromfile(file_name, dtype=self.info_interface.get_channel_datatype(channel_name))
   cuboid_data = cuboid_data.reshape(dimensions)
   super_zidx = XYZMorton([x_index, y_index, z_index])
   self.logger.info("Inserting cube {},{},{}".format(x_index, y_index, z_index))
   self.cuboidindex_db.putItem(channel_name, resolution, x_index, y_index, z_index, time_index, neariso=neariso)
   self.cuboid_bucket.putObject(channel_name, resolution, super_zidx, time_index, blosc.pack_array(cuboid_data), neariso=neariso)
Ejemplo n.º 29
0
 def toBlosc ( self ):
   """Pack the object"""
   try:
     # Create the compressed cube
     return blosc.pack_array(self.data) 
   except:
     logger.error ("Failed to compress database cube.  Data integrity concern.")
     raise
Ejemplo n.º 30
0
  def putIndex ( self, ch, entityid, timestamp, resolution, index, update=False ):
    """Write the index for the annotation with id"""

    if self.NPZ:
      fileobj = cStringIO.StringIO ()
      np.save ( fileobj, index )
      self.kvio.putIndex(ch, entityid, timestamp, resolution, fileobj.getvalue(), update )
    else:
      self.kvio.putIndex(ch, entityid, timestamp, resolution, blosc.pack_array(index), update)
Ejemplo n.º 31
0
    def test_channel_uint64_cuboid_unaligned_offset_time_offset_blosc_numpy(
            self):
        """ Test uint64 data, not cuboid aligned, offset, time samples, blosc interface

        Test Requires >=2GB of memory!
        """

        test_mat = np.random.randint(1, 2**50, (3, 17, 300, 500))
        test_mat = test_mat.astype(np.uint64)
        bb = blosc.pack_array(test_mat)

        # Create request
        factory = APIRequestFactory()
        request = factory.post(
            '/' + version +
            '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203',
            bb,
            content_type='application/blosc-python')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request,
                                    collection='col1',
                                    experiment='exp1',
                                    dataset='layer1',
                                    resolution='0',
                                    x_range='100:600',
                                    y_range='450:750',
                                    z_range='20:37')
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)

        # Create Request to get data you posted
        request = factory.get(
            '/' + version +
            '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203',
            HTTP_ACCEPT='application/blosc-python')

        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request,
                                    collection='col1',
                                    experiment='exp1',
                                    dataset='layer1',
                                    resolution='0',
                                    x_range='100:600',
                                    y_range='450:750',
                                    z_range='20:37').render()
        self.assertEqual(response.status_code, status.HTTP_200_OK)

        # Decompress
        data_mat = blosc.unpack_array(response.content)

        # Test for data equality (what you put in is what you got back!)
        np.testing.assert_array_equal(data_mat, test_mat)
Ejemplo n.º 32
0
 def toBlosc(self):
     """Pack the object"""
     try:
         # Create the compressed cube
         return blosc.pack_array(self.data)
     except:
         logger.error(
             "Failed to compress database cube.  Data integrity concern.")
         raise
Ejemplo n.º 33
0
  def putIndex ( self, ch, entityid, resolution, index, update=False ):
    """Write the index for the annotation with id"""

    if self.NPZ:
      fileobj = cStringIO.StringIO ()
      np.save ( fileobj, index )
      self.kvio.putIndex(ch, entityid, resolution, fileobj.getvalue(), update)
    else:
      self.kvio.putIndex(ch, entityid, resolution, blosc.pack_array(index), update)
Ejemplo n.º 34
0
def generate_dagger_demos(env_name, seeds, fail_obss, fail_actions, mean_steps):
    env = gym.make(env_name)
    agent = BotAgent(env)
    demos = []

    for i in range(len(fail_obss)):
        # Run the expert for one episode
        env.seed(int(seeds[i]))

        new_obs = env.reset()
        agent.on_reset()

        env0_str = env.__str__()

        actions = []
        images = []
        directions = []
        debug_info = {'seed': [int(seeds[i])], 'actions': []}
        try:
            for j in range(min(int(args.dagger_trim_coef * mean_steps), len(fail_obss[i]) - 1)):
                obs = fail_obss[i][j]
                assert check_obss_equality(obs, new_obs), "Observations {} of seed {} don't match".format(j, seeds[i])
                mission = obs['mission']
                action = agent.act(update_internal_state=False)['action']
                _ = agent.bot.take_action(fail_actions[i][j])
                debug_info['actions'].append(fail_actions[i][j])
                new_obs, reward, done, _ = env.step(fail_actions[i][j])
                if done and reward > 0:
                    raise ValueError(
                        "The baby's actions shouldn't solve the task. Env0 {}, Env9{}, Seed {}, actions {}.".format(
                            env0_str, env.__str__(), int(seeds[i]), fail_actions[i]
                        ))
                actions.append(action)
                images.append(obs['image'])
                directions.append(obs['direction'])
            if args.continue_dagger:
                obs = new_obs
                while not done:
                    action = agent.act(obs)['action']
                    debug_info['actions'].append(action)
                    new_obs, reward, done, _ = env.step(action)
                    agent.analyze_feedback(reward, done)
                    actions.append(action)
                    images.append(obs['image'])
                    directions.append(obs['direction'])
            print(debug_info, actions)

            demos.append((mission, blosc.pack_array(np.array(images)), directions, actions))

        except Exception as e:
            logger.exception("error while generating demo #{}: {}. Env0 {}, Env9{}, Seed {}, actions {}.".format(
                len(demos), e, env0_str, env.__str__(), int(seeds[i]), fail_actions[i]))
            continue

    return demos
Ejemplo n.º 35
0
    def test_channel_uint16_cuboid_aligned_offset_no_time_blosc_numpy(self):
        """ Test uint16 data, cuboid aligned, offset, no time samples, blosc interface"""

        test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128))
        test_mat = test_mat.astype(np.uint16)
        bb = blosc.pack_array(test_mat)

        # Create request
        factory = APIRequestFactory()
        request = factory.post(
            '/' + version +
            '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/',
            bb,
            content_type='application/blosc-python')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request,
                                    collection='col1',
                                    experiment='exp1',
                                    channel='channel2',
                                    resolution='0',
                                    x_range='128:256',
                                    y_range='256:384',
                                    z_range='16:32',
                                    t_range=None)
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)

        # Create Request to get data you posted
        request = factory.get(
            '/' + version +
            '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/',
            HTTP_ACCEPT='application/blosc-python')

        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request,
                                    collection='col1',
                                    experiment='exp1',
                                    channel='channel2',
                                    resolution='0',
                                    x_range='128:256',
                                    y_range='256:384',
                                    z_range='16:32',
                                    t_range=None).render()
        self.assertEqual(response.status_code, status.HTTP_200_OK)

        # Decompress
        data_mat = blosc.unpack_array(response.content)

        # Test for data equality (what you put in is what you got back!)
        np.testing.assert_array_equal(data_mat, test_mat)
Ejemplo n.º 36
0
def blosc_up():
    for file in file_list:
        img_name=photo_test+'\\'+file
        img=Image.open(img_name)
        # numpy화
        # numpy.asarry() 이미지->array
        image_array=np.asarray(img)
        #  pack_array 배열 압축
        compressed_bytes = blosc.pack_array(image_array)
        xrayimg={"imagenum":file, "photo":compressed_bytes}
        client.kaggle1.blosc.insert_one(xrayimg)
Ejemplo n.º 37
0
 def mergeCubes(data1, data2):
   """Merge Cubes"""
   
   import ndlib 
   data1 = blosc.unpack_array(data1)
   data2 = blosc.unpack_array(data2)
   # Call vectorize function
   # vec_func = np.vectorize(lambda x,y: x if y == 0 else y)
   # Call ctype function
   ndlib.overwriteMerge_ctype(data1, data2)
   return blosc.pack_array(data1)
Ejemplo n.º 38
0
    def test_put_object(self):
        """Testing put object"""
        cube_data = blosc.pack_array(np.zeros(settings.SUPER_CUBOID_SIZE))
        for morton_index in range(0, 10, 1):
            self.cuboid_bucket.putObject(nd_proj.channel_name,
                                         nd_proj.resolution, morton_index,
                                         cube_data)

        for morton_index in range(0, 10, 1):
            supercuboid_key = self.cuboid_bucket.generateSupercuboidKey(
                nd_proj.channel_name, nd_proj.resolution, morton_index)
            self.cuboid_bucket.deleteObject(supercuboid_key)
Ejemplo n.º 39
0
    def test_put_object_by_key(self):
        hashm = hashlib.md5()
        hashm.update(b"test_cuboidbucket_data")
        cube_data = blosc.pack_array(np.zeros(settings.SUPER_CUBOID_SIZE))

        for morton_index in range(0, 10, 1):
            key = "{}&{}".format(hashm.hexdigest(), morton_index)
            self.cuboid_bucket.putObjectByKey(key, cube_data)

        for morton_index in range(0, 10, 1):
            key = "{}&{}".format(hashm.hexdigest(), morton_index)
            self.cuboid_bucket.deleteObject(key)
Ejemplo n.º 40
0
def PostBlosc (p, post_data):
  """Post data using the npz interface"""
  
  # Build the url and then create a npz object
  url = 'http://{}/blaze/{}/{}/blosc/{}/{},{}/{},{}/{},{}/'.format(SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args)

  try:
    # Build a post request
    req = urllib2.Request(url, blosc.pack_array(post_data))
    response = urllib2.urlopen(req)
    return response
  except urllib2.HTTPError,e:
    return e
Ejemplo n.º 41
0
    def test_param_constructor(self):
        """Re-run a testing using the parameter based constructor"""
        config = {
            "cache_host": self.config["aws"]["cache"],
            "cache_db": 1,
            "read_timeout": 86400
        }
        rkv = RedisKVIO(config)

        # Clean up data
        self.cache_client.flushdb()

        data1 = np.random.randint(50, size=[10, 15, 5])
        data2 = np.random.randint(50, size=[10, 15, 5])
        data3 = np.random.randint(50, size=[10, 15, 5])
        data_packed1 = blosc.pack_array(data1)
        data_packed2 = blosc.pack_array(data2)
        data_packed3 = blosc.pack_array(data3)
        data = [data_packed1, data_packed2, data_packed3]

        # Add items
        morton_id = [112, 125, 516]
        keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
                                               morton_id)
        rkv.put_cubes(keys, data)

        # Get cube
        cubes = rkv.get_cubes(keys)

        cube = [x for x in cubes]

        assert len(cube) == 3

        for m, c, d in zip(morton_id, cube, data):
            assert c[0] == m
            assert c[1] == 0
            data_retrieved = blosc.unpack_array(c[2])
            np.testing.assert_array_equal(data_retrieved,
                                          blosc.unpack_array(d))
Ejemplo n.º 42
0
 def parse(self, response):
     for article in response.css("article.post"):
         for image in article.css("img.attachment-post-thumbnail"):
             with urllib.request.urlopen(
                     image.xpath('@src').get()) as img_res:
                 image_array = np.asarray(bytearray(img_res.read()),
                                          dtype=np.uint8)
                 compressed_img = blosc.pack_array(image_array)
             yield {
                 'id': article.attrib['id'],
                 'src': image.xpath('@src').get(),
                 'img': compressed_img
             }
Ejemplo n.º 43
0
    def test_param_constructor(self):
        """Re-run a testing using the parameter based constructor"""
        config = {
                    "cache_host": self.config["aws"]["cache"],
                    "cache_db": 1,
                    "read_timeout": 86400
                }
        rkv = RedisKVIO(config)

        # Clean up data
        self.cache_client.flushdb()

        data1 = np.random.randint(50, size=[10, 15, 5])
        data2 = np.random.randint(50, size=[10, 15, 5])
        data3 = np.random.randint(50, size=[10, 15, 5])
        data_packed1 = blosc.pack_array(data1)
        data_packed2 = blosc.pack_array(data2)
        data_packed3 = blosc.pack_array(data3)
        data = [data_packed1, data_packed2, data_packed3]

        # Add items
        morton_id = [112, 125, 516]
        keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)
        rkv.put_cubes(keys, data)

        # Get cube
        cubes = rkv.get_cubes(keys)

        cube = [x for x in cubes]

        assert len(cube) == 3

        for m, c, d in zip(morton_id, cube, data):
            assert c[0] == m
            assert c[1] == 0
            data_retrieved = blosc.unpack_array(c[2])
            np.testing.assert_array_equal(data_retrieved, blosc.unpack_array(d))
Ejemplo n.º 44
0
    def breakCubes(key, blosc_data):
      """break the cubes into smaller chunks"""
      
      key_array = [token, channel_name, res, x1, x2, y1, y2, z1, z2, time_stamp] = key.split('_')
      [res, x1, x2, y1, y2, z1, z2] = [int(i) for i in key_array[2:][:-1]]
      if blosc_data is None:
        return
      voxarray = blosc.unpack_array(blosc_data)
      
      br = BlazeRedis(token, channel_name, res)

      ds = Dataset(token)
      ch = ds.getChannelObj(channel_name)
      [zimagesz, yimagesz, ximagesz] = ds.imagesz[res]
      #[xcubedim, ycubedim, zcubedim] = cubedim = ds.cubedim[res]
      [xcubedim, ycubedim, zcubedim] = cubedim = CUBE_DIM
      [xoffset, yoffset, zoffset] = ds.offset[res]
      
      # Calculating the corner and dimension
      corner = [x1, y1, z1]
      dim = voxarray.shape[::-1][:-1]

      # Round to the nearest largest cube in all dimensions
      [xstart, ystart, zstart] = start = map(div, corner, cubedim)

      znumcubes = (corner[2]+dim[2]+zcubedim-1)/zcubedim - zstart
      ynumcubes = (corner[1]+dim[1]+ycubedim-1)/ycubedim - ystart
      xnumcubes = (corner[0]+dim[0]+xcubedim-1)/xcubedim - xstart
      numcubes = [xnumcubes, ynumcubes, znumcubes]
      offset = map(mod, corner, cubedim)

      data_buffer = np.zeros(map(mul, numcubes, cubedim)[::-1], dtype=voxarray.dtype)
      end = map(add, offset, dim)
      data_buffer[offset[2]:end[2], offset[1]:end[1], offset[0]:end[0]] = voxarray

      cube_list = []
      for z in range(znumcubes):
        for y in range(ynumcubes):
          for x in range(xnumcubes):
            zidx = XYZMorton(map(add, start, [x,y,z]))
           
            # Parameters in the cube slab
            index = map(mul, cubedim, [x,y,z])
            end = map(add, index, cubedim)

            cube_data = data_buffer[index[2]:end[2], index[1]:end[1], index[0]:end[0]]
            cube_list.append((br.generateSIKey(zidx), blosc.pack_array(cube_data.reshape((1,)+cube_data.shape))))
      
      return cube_list[:]
Ejemplo n.º 45
0
    def test_write_buffer_io(self):
        """Test methods specific to single cube write buffer io"""
        resolution = 1
        rkv = RedisKVIO(self.config_data)

        # Clean up data
        self.cache_client.flushdb()

        data1 = np.random.randint(50, size=[10, 15, 5])
        data_packed = blosc.pack_array(data1)

        key = rkv.insert_cube_in_write_buffer("WRITE-CUBOID&4&1&1&1", 3, 234, data_packed)

        data_rx = rkv.get_cube_from_write_buffer(key)

        assert data_packed == data_rx
Ejemplo n.º 46
0
def from_array(array):
    """
    Export a numpy array to a blosc array.

    Arguments:
        array: The numpy array to compress to blosc array

    Returns:
        Bytes/String. A blosc compressed array
    """
    try:
        raw_data = blosc.pack_array(array)
    except Exception as e:
        raise ValueError("Could not compress data from array. {}".format(e))

    return raw_data
Ejemplo n.º 47
0
def generate_demos(env_name, seeds):
    env = gym.make(env_name)

    raise(NotImplementedError, 'RL expert goes here')
    #TODO: insert neural agent here (RL expert) (Mathijs)

    #agent = BotAgent(env)

    demos = []

    for seed in seeds:
        # Run the expert for one episode
        done = False

        env.seed(int(seed))
        obs = env.reset()
        agent.on_reset()

        actions = []
        mission = obs["mission"]
        images = []
        directions = []

        try:
            while not done:
                action = agent.act(obs)['action']
                new_obs, reward, done, _ = env.step(action)
                agent.analyze_feedback(reward, done)

                actions.append(action)
                images.append(obs['image'])
                directions.append(obs['direction'])

                obs = new_obs

            if reward > 0:
                demos.append((mission, blosc.pack_array(np.array(images)), directions, actions))
            if reward == 0:
                logger.info("failed to accomplish the mission")

        except Exception:
            logger.exception("error while generating demo #{}".format(len(demos)))
            continue

        # logger.info("demo #{}".format(len(demos)))

    return demos
Ejemplo n.º 48
0
 def multiThreadTest(self, start_value, size_iterations, number_of_processes):
   """Generate the URL for multi-thread test"""
   
   # min_values = [xmin,ymin,zmin] = map(add, self.info_interface.offset(self.resolution), start_value)
   min_values = [xmin, ymin, zmin] = self.info_interface.offset(self.resolution)
   # max_values = map(add, min_values, self.info_interface.cuboid_dimension(self.resolution))
   max_values = map(add, min_values, map(lambda x: x*self.scale, self.info_interface.supercuboid_dimension(self.resolution)))
   range_args = [None]*(len(min_values)+len(max_values))
   size_args = [None]*(len(min_values)+len(max_values))
   
   # determine the size of the cutout
   for i in range(0, size_iterations, 1):
     if all([a<b for a,b in zip(max_values, self.info_interface.image_size(self.resolution))]):
       size_args[::2] = min_values
       size_args[1::2] = max_values
       # increase in x,y
       max_values[(i+1)%2] = (max_values[(i+1)%2]-min_values[(i+1)%2])*2
     else:
       break
   
   # intialize the range args
   range_args[:] = size_args[:]
   range_args[::2] = map(add, start_value, size_args[::2])
   range_args[1::2] = map(add, start_value, size_args[1::2])
   
   # creating a fetch list for each process
   for i in range(0, number_of_processes, 1):
     # checking if the x,y,z dimensions are exceeded
     if all([a<b for a,b in zip(range_args[1::2], self.info_interface.image_size(self.resolution))]):
       if self.write_tests:
         data = blosc.pack_array(np.ones( [len(self.channels)]+map(sub, range_args[1::2], range_args[::2])[::-1], dtype=self.datatype) * random.randint(0,255))
         self.data_list.append(data)
         self.fetch_list.append(generateURLBlosc(self.host_name, self.token_name, self.channels, self.resolution, range_args, direct=self.direct))
       else:
         self.fetch_list.append(generateURLBlosc(self.host_name, self.token_name, self.channels, self.resolution, range_args, direct=self.direct))
       # checking if this exceeds the x,y image size
       if all([a<b for a,b in zip(map(add, range_args[1:-1:2], size_args[1:-1:2]), self.info_interface.image_size(self.resolution))]): 
         range_args[0:-2:2] = range_args[1:-1:2]
         range_args[1:-1:2]  = map(add, range_args[1:-1:2], size_args[1:-1:2])
       # if you cannot expand more in x,y then expand in z
       elif range_args[-1] < self.info_interface.image_size(self.resolution)[-1]:
         range_args[-2] = range_args[-1]
         range_args[-1] = range_args[-1] + size_args[-1]
       else:
         break
     else:
       break
Ejemplo n.º 49
0
    def test_channel_uint16_wrong_data_type_numpy(self):
        """ Test posting the wrong bitdepth data using the blosc-numpy interface"""
        test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128))
        test_mat = test_mat.astype(np.uint8)
        bb = blosc.pack_array(test_mat)

        # Create request
        factory = APIRequestFactory()
        request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/0:128/0:128/0:16/', bb,
                               content_type='application/blosc-python')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2',
                                    resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None)
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
Ejemplo n.º 50
0
def postBlaze (p, post_data, time=False):
  """Post data using npz"""

  # Build the url and then create a npz object
  if time:
    url = 'https://{}/sd/{}/{}/blaze/{}/{},{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
  elif p.channels is not None:
    url = 'https://{}/sd/{}/{}/blaze/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
  elif p.channels is None:
    url = 'https://{}/sd/{}/blaze/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, p.resolution, *p.args )

  try:
    # Build a post request
    resp = postURL(url, blosc.pack_array(post_data))
    return resp
  except Exception as e:
    return e
Ejemplo n.º 51
0
def postBlosc (p, post_data, time=False):
  """Post data using npz"""
  
  # Build the url and then create a npz object
  if time:
    url = 'http://{}/ca/{}/{}/blosc/{}/{},{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
  elif p.channels is not None:
    url = 'http://{}/ca/{}/{}/blosc/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
  elif p.channels is None:
    url = 'http://{}/ca/{}/blosc/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, p.resolution, *p.args )

  try:
    # Build a post request
    req = urllib2.Request(url,blosc.pack_array(post_data))
    response = urllib2.urlopen(req)
    return response
  except urllib2.HTTPError,e:
    return e
Ejemplo n.º 52
0
    def test_channel_uint64_time_npygz_download(self):
        """ Test uint8 data, using the npygz interface with time series support

        """
        test_mat = np.random.randint(1, 256, (3, 4, 128, 128))
        test_mat = test_mat.astype(np.uint64)
        bb = blosc.pack_array(test_mat)

        # Create request
        factory = APIRequestFactory()
        request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13', bb,
                               content_type='application/blosc-python')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1',
                                    resolution='0', x_range='0:128', y_range='0:128', z_range='0:4',
                                    t_range='10:13')
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)

        # Create Request to get data you posted
        request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13',
                              HTTP_ACCEPT='application/npygz')

        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1',
                                    resolution='0', x_range='0:128', y_range='0:128', z_range='0:4',
                                    t_range='10:13').render()
        self.assertEqual(response.status_code, status.HTTP_200_OK)

        # Decompress
        data_bytes = zlib.decompress(response.content)

        # Open
        data_obj = io.BytesIO(data_bytes)
        data_mat = np.load(data_obj)

        # Test for data equality (what you put in is what you got back!)
        np.testing.assert_array_equal(data_mat, test_mat)
Ejemplo n.º 53
0
 def insert_tile(self, zoom, row, col, data):
     with self.db_connection as db_connection:
         cursor = db_connection.cursor()
         compression = self.compression
         data_type = self.data_type
         if compression and (data_type == "xray"):
             db_connection.text_factory = str
             data_compressed = blosc.pack_array(data, cname=compression)
             data = data_compressed
         if data_type == "image/TIFF":
             try:
                 assert data.dtype == "uint8"
             except:
                 raise TypeError("dtype %s not supported" % data.dtype)
             image = Image.fromarray(np.uint8(data))
             buf = ioBuffer()
             if compression == "tiff_lzw":
                 TiffImagePlugin.WRITE_LIBTIFF = True
                 image.save(buf, "TIFF", compression=compression)
                 TiffImagePlugin.WRITE_LIBTIFF = False
             else:
                 image.save(buf, "TIFF", compression=compression)
             buf.seek(0)
             data = Binary(buf.read())
         if data_type == "image/JPEG2000":
             try:
                 assert data.dtype == "uint8"
             except:
                 raise TypeError("dtype %s not supported" % data.dtype)
             image = Image.fromarray(np.uint8(data))
             buf = ioBuffer()
             image.save(buf, "j2k")
             buf.seek(0)
             data = Binary(buf.read())
         try:
             cursor.execute("""
                 INSERT INTO tiles
                     (zoom_level, tile_row, tile_column, tile_data)
                     VALUES (?,?,?,?)
             """, (zoom, row, col, data))
         except:
             raise
Ejemplo n.º 54
0
    def test_channel_uint64_cuboid_unaligned_offset_time_offset_blosc_numpy(self):
        """ Test uint64 data, not cuboid aligned, offset, time samples, blosc interface

        Test Requires >=2GB of memory!
        """

        test_mat = np.random.randint(1, 256, (3, 17, 300, 500))
        test_mat = test_mat.astype(np.uint64)
        bb = blosc.pack_array(test_mat)

        # Create request
        factory = APIRequestFactory()
        request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203', bb,
                               content_type='application/blosc-python')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1',
                                    resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='200:203')
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)

        # Create Request to get data you posted
        request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203',
                              HTTP_ACCEPT='application/blosc-python')

        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1',
                                    resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='200:203').render()
        self.assertEqual(response.status_code, status.HTTP_200_OK)

        # Decompress
        data_mat = blosc.unpack_array(response.content)

        # Test for data equality (what you put in is what you got back!)
        np.testing.assert_array_equal(data_mat, test_mat)
Ejemplo n.º 55
0
def postBlosc (p, post_data, time=False, neariso=False, direct=False):
  """Post data using blosc packed numpy array"""

  # Build the url and then create a npz object
  if time:
    url = 'https://{}/sd/{}/{}/blosc/{}/{},{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
  elif p.channels is not None:
    url = 'https://{}/sd/{}/{}/blosc/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
  elif p.channels is None:
    url = 'https://{}/sd/{}/blosc/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, p.resolution, *p.args )

  if neariso:
    url = url + NEARISO

  if direct:
    url = url + DIRECT
  
  try:
    # Build a post request
    resp = postURL(url,blosc.pack_array(post_data))
    return resp
  except Exception as e:
    return e
Ejemplo n.º 56
0
    def test_channel_uint16_cuboid_aligned_offset_no_time_blosc_numpy(self):
        """ Test uint16 data, cuboid aligned, offset, no time samples, blosc interface"""

        test_mat = np.random.randint(1, 2**16-1, (16, 128, 128))
        test_mat = test_mat.astype(np.uint16)
        bb = blosc.pack_array(test_mat)

        # Create request
        factory = APIRequestFactory()
        request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/', bb,
                               content_type='application/blosc-python')
        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2',
                                    resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None)
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)

        # Create Request to get data you posted
        request = factory.get('/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/',
                              HTTP_ACCEPT='application/blosc-python')

        # log in user
        force_authenticate(request, user=self.user)

        # Make request
        response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2',
                                    resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None).render()
        self.assertEqual(response.status_code, status.HTTP_200_OK)

        # Decompress
        data_mat = blosc.unpack_array(response.content)

        # Test for data equality (what you put in is what you got back!)
        np.testing.assert_array_equal(data_mat, test_mat)