def test_get_object_bounding_box_single_cuboid(self): """ Test getting the bounding box of a object""" test_mat = np.ones((128, 128, 16)) test_mat[0:128, 0:128, 0:16] = 4 test_mat = test_mat.astype(np.uint64) test_mat = test_mat.reshape((16, 128, 128)) bb = blosc.compress(test_mat, typesize=64) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/bbchan1/0/1536:1664/1536:1664/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='bbchan1', resolution='0', x_range='1536:1664', y_range='1536:1664', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/bbchan1/0/1536:1664/1536:1664/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='bbchan1', resolution='0', x_range='1536:1664', y_range='1536:1664', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint64) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat) # get the bounding box # Create request factory = APIRequestFactory() request = factory.get('/' + version + '/boundingbox/col1/exp1/bbchan1/0/4') # log in user force_authenticate(request, user=self.user) # Make request response = BoundingBox.as_view()(request, collection='col1', experiment='exp1', channel='bbchan1', resolution='0', id='4') self.assertEqual(response.status_code, status.HTTP_200_OK) bb = response.data self.assertEqual(bb['t_range'], [0, 1]) self.assertEqual(bb['x_range'], [1536, 2048]) self.assertEqual(bb['y_range'], [1536, 2048]) self.assertEqual(bb['z_range'], [0, 16])
def test_channel_uint16_cuboid_unaligned_offset_time_blosc(self): """ Test uint16 data, not cuboid aligned, offset, time samples, blosc interface Test Requires >=2GB of memory! """ test_mat = np.random.randint(1, 2**16 - 1, (3, 17, 300, 500)) test_mat = test_mat.astype(np.uint16) h = test_mat.tobytes() bb = blosc.compress(h, typesize=16) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel2/0/100:600/450:750/20:37/0:3', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', dataset='channel2', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get( '/' + version + '/cutout/col1/exp1/channel2/0/100:600/450:750/20:37/0:3', HTTP_ACCEPT='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', dataset='channel2', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint16) data_mat = np.reshape(data_mat, (3, 17, 300, 500), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint16_cuboid_aligned_offset_no_time_blosc(self): """ Test uint16 data, cuboid aligned, offset, no time samples, blosc interface""" test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint16) h = test_mat.tobytes() bb = blosc.compress(h, typesize=16) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get( '/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint16) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint64_cuboid_unaligned_offset_time_offset_blosc_numpy( self): """ Test uint64 data, not cuboid aligned, offset, time samples, blosc interface Test Requires >=2GB of memory! """ test_mat = np.random.randint(1, 2**50, (3, 17, 300, 500)) test_mat = test_mat.astype(np.uint64) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', dataset='layer1', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get( '/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203', HTTP_ACCEPT='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', dataset='layer1', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_mat = blosc.unpack_array(response.content) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint64_cuboid_aligned_no_offset_no_time_blosc(self): """ Test uint64 data, cuboid aligned, no offset, no time samples""" test_mat = np.ones((128, 128, 16)) test_mat = test_mat.astype(np.uint64) test_mat = test_mat.reshape((16, 128, 128)) bb = blosc.compress(test_mat, typesize=64) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint64) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat) # get the bounding box # Create request factory = APIRequestFactory() request = factory.get('/' + version + '/ids/col1/exp1/layer1/0/0:128/0:128/0:16/') # log in user force_authenticate(request, user=self.user) # Make request response = Ids.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data['ids'], ['1'])
def test_channel_uint16_cuboid_aligned_no_offset_no_time_blosc_numpy(self): """ Test uint16 data, cuboid aligned, no offset, no time samples""" test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint16) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel2/0/0:128/0:128/0:16/', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel2/0/0:128/0:128/0:16/', HTTP_ACCEPT='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_mat = blosc.unpack_array(response.content) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint64_time_npygz_upload(self): """ Test uint8 data, using the npygz interface with time series support while uploading in that format as well """ test_mat = np.random.randint(1, 256, (3, 4, 128, 128)) test_mat = test_mat.astype(np.uint64) # Save Data to npy npy_file = io.BytesIO() np.save(npy_file, test_mat, allow_pickle=False) # Compress npy npy_gz = zlib.compress(npy_file.getvalue()) # Send file npy_gz_file = io.BytesIO(npy_gz) npy_gz_file.seek(0) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13', npy_gz_file.read(), content_type='application/npygz') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:4', t_range='10:13') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13', HTTP_ACCEPT='application/npygz') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:4', t_range='10:13').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_bytes = zlib.decompress(response.content) # Open data_obj = io.BytesIO(data_bytes) data_mat = np.load(data_obj) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint16_wrong_dimensions_numpy(self): """ Test posting with the wrong xyz dims using the numpy interface""" test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint16) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel2/0/0:100/0:128/0:16/', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:100', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def setUp(self): """ Copy params from the Layer setUpClass """ # Setup config self.kvio_config = self.layer.kvio_config self.state_config = self.layer.state_config self.object_store_config = self.layer.object_store_config self.user = self.layer.user # Log Django User in self.client.force_login(self.user) if not self.test_data_loaded: # Flush cache between tests client = redis.StrictRedis(host=self.kvio_config['cache_host'], port=6379, db=1, decode_responses=False) client.flushdb() client = redis.StrictRedis(host=self.state_config['cache_state_host'], port=6379, db=1, decode_responses=False) client.flushdb() # load some data for reading self.test_data_8 = np.random.randint(1, 254, (16, 1024, 1024), dtype=np.uint8) bb = blosc.compress(self.test_data_8, typesize=8) # Post data to the database factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel1/0/0:1024/0:1024/0:16/', bb, content_type='application/blosc') force_authenticate(request, user=self.user) _ = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:1024', y_range='0:1024', z_range='0:16', t_range=None) self.test_data_loaded = True
def test_channel_uint16_wrong_data_type(self): """ Test posting the wrong bitdepth data """ test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint8) h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel2/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_channel_uint16_get_too_big(self): """ Test getting a cutout that is over 1GB uncompressed""" # Create request factory = APIRequestFactory() # Create Request to get data you posted request = factory.get( '/' + version + '/cutout/col1/exp1/channel2/0/0:100000/0:100000/0:10000/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:100000', y_range='0:100000', z_range='0:10000', t_range=None) self.assertEqual(response.status_code, status.HTTP_413_REQUEST_ENTITY_TOO_LARGE)
def test_channel_uint64_filter_ids_not_found(self): """ Test filter_cutout by ids not in the region""" test_mat = np.ones((128, 128, 4)) test_mat[0][0][0] = 2 test_mat[0][0][1] = 3 test_mat[0][0][2] = 4 test_mat = test_mat.reshape(4, 128, 128) test_mat = test_mat.astype(np.uint64) h = test_mat.tobytes() bb = blosc.compress(h, typesize=64) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/128:256/256:384/16:20/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='128:256', y_range='256:384', z_range='16:20', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/128:256/256:384/16:20/?filter=5,6,7', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='128:256', y_range='256:384', z_range='16:20', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint64) data_mat = np.reshape(data_mat, (4, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(np.unique(data_mat), np.arange(0, 1, dtype=np.uint64))
def test_channel_uint64_time_npygz_download(self): """ Test uint8 data, using the npygz interface with time series support """ test_mat = np.random.randint(1, 256, (3, 4, 128, 128)) test_mat = test_mat.astype(np.uint64) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:4', t_range='10:13') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13', HTTP_ACCEPT='application/npygz') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:4', t_range='10:13').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_bytes = zlib.decompress(response.content) # Open data_obj = io.BytesIO(data_bytes) data_mat = np.load(data_obj) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint16_cuboid_unaligned_offset_time_blosc(self): """ Test uint16 data, not cuboid aligned, offset, time samples, blosc interface Test Requires >=2GB of memory! """ test_mat = np.random.randint(1, 2**16-1, (3, 17, 300, 500)) test_mat = test_mat.astype(np.uint16) h = test_mat.tobytes() bb = blosc.compress(h, typesize=16) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/100:600/450:750/20:37/0:3', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='0:3') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel2/0/100:600/450:750/20:37/0:3', HTTP_ACCEPT='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='0:3').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint16) data_mat = np.reshape(data_mat, (3, 17, 300, 500), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint64_cuboid_aligned_offset_no_time_blosc(self): """ Test uint64 data, cuboid aligned, offset, no time samples, blosc interface""" test_mat = np.random.randint(1, 256, (4, 128, 128)) test_mat = test_mat.astype(np.uint64) h = test_mat.tobytes() bb = blosc.compress(h, typesize=64) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/128:256/256:384/16:20/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='128:256', y_range='256:384', z_range='16:20', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/128:256/256:384/16:20/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='128:256', y_range='256:384', z_range='16:20', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint64) data_mat = np.reshape(data_mat, (4, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint64_cuboid_unaligned_offset_time_offset_blosc_numpy(self): """ Test uint64 data, not cuboid aligned, offset, time samples, blosc interface Test Requires >=2GB of memory! """ test_mat = np.random.randint(1, 256, (3, 17, 300, 500)) test_mat = test_mat.astype(np.uint64) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='200:203') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203', HTTP_ACCEPT='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='200:203').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_mat = blosc.unpack_array(response.content) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint16_cuboid_aligned_offset_no_time_blosc_numpy(self): """ Test uint16 data, cuboid aligned, offset, no time samples, blosc interface""" test_mat = np.random.randint(1, 2**16-1, (16, 128, 128)) test_mat = test_mat.astype(np.uint16) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/', HTTP_ACCEPT='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_mat = blosc.unpack_array(response.content) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint16_get_too_big(self): """ Test getting a cutout that is over 1GB uncompressed""" # Create request factory = APIRequestFactory() # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel2/0/0:100000/0:100000/0:10000/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:100000', y_range='0:100000', z_range='0:10000', t_range=None) self.assertEqual(response.status_code, status.HTTP_413_REQUEST_ENTITY_TOO_LARGE)
def test_channel_uint16_wrong_data_type_numpy(self): """ Test posting the wrong bitdepth data using the blosc-numpy interface""" test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint8) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/0:128/0:128/0:16/', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_channel_uint16_wrong_dimensions(self): """ Test posting with the wrong xyz dims""" test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint16) h = test_mat.tobytes() bb = blosc.compress(h, typesize=16) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/0:100/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:100', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_channel_uint8_cuboid_aligned_offset(self): "uint8 - cuboid aligned - offset present" # Get test data from cloudvolume, returned in XYZT test_mat = self.vol_uint8[0:512, 0:512, 16:32] # Convert to ZYX test_mat = np.squeeze(test_mat).T # Create Request to get data you posted factory = APIRequestFactory() request = factory.get( '/' + version + '/cutout/col1/exp1/chan1/0/128:256/256:384/16:32/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='chan1', resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint8) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) offset_mat = test_mat[:, 256:384, 128:256] np.testing.assert_array_equal(data_mat, offset_mat)
def test_delayed_write(self): """A test with multiple writes to the same cuboid""" test_mat = np.random.randint(1, 254, (16, 128, 128)) test_mat = test_mat.astype(np.uint8) h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) test_mat[1, 20, 40] = 5 test_mat[1, 20, 41] = 5 test_mat[1, 20, 42] = 5 h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint8) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_delayed_write(self): """A test with multiple writes to the same cuboid""" test_mat = np.random.randint(1, 254, (16, 128, 128)) test_mat = test_mat.astype(np.uint8) h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) test_mat[1, 20, 40] = 5 test_mat[1, 20, 41] = 5 test_mat[1, 20, 42] = 5 h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint8) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_start_downsample_get_status_and_check_data(self): """A large complex test that verifies all the pluming for downsample. Does not validate data integrity, but does make sure data exists at different levels and iso vs. aniso.""" self.dbsetup.insert_downsample_data() # Post some data to the channel test_mat = np.random.randint(1, 254, (16, 1024, 1024)) test_mat = test_mat.astype(np.uint8) h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/0/0:1024/0:1024/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='0', x_range='0:1024', y_range='0:1024', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Wait for data to be written request = factory.get( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/0/0:1024/0:1024/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='0', x_range='0:1024', y_range='0:1024', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Trigger downsample factory = APIRequestFactory() request = factory.post('/' + version + '/downsample/col1/exp_ds_aniso/channel1/', content_type='application/json') # log in user force_authenticate(request, user=self.user) # Make request response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Make Sure status has changed factory = APIRequestFactory() request = factory.get('/' + version + '/downsample/col1/exp_ds_aniso/channel1/', content_type='application/json') # log in user force_authenticate(request, user=self.user) # Make request response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1').render() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["num_hierarchy_levels"], 5) self.assertEqual(response.data["status"], "IN_PROGRESS") for _ in range(0, 30): # Make request response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1').render() self.assertEqual(response.status_code, status.HTTP_200_OK) if response.data["status"] != "IN_PROGRESS": break time.sleep(2) # Verify now downsampled response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1').render() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["num_hierarchy_levels"], 5) self.assertEqual(response.data["status"], "DOWNSAMPLED") # Get data at res 1 and verify it's non-zero request = factory.get( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/1/0:512/0:512/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='1', x_range='0:512', y_range='0:512', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) raw_data = blosc.decompress(response.content) data_mat_res1_aniso = np.fromstring(raw_data, dtype=np.uint8) data_mat_res1_aniso = np.reshape(data_mat_res1_aniso, (16, 512, 512), order='C') # Make sure not blank self.assertGreater(data_mat_res1_aniso.sum(), 100) # Get data at res 1 with iso flag and verify it's non-zero and the same as without flag request = factory.get( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/1/0:512/0:512/0:16/?iso=true', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='1', x_range='0:512', y_range='0:512', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) raw_data = blosc.decompress(response.content) data_mat_res1_iso = np.fromstring(raw_data, dtype=np.uint8) data_mat_res1_iso = np.reshape(data_mat_res1_iso, (16, 512, 512), order='C') # Make sure not blank self.assertGreater(data_mat_res1_iso.sum(), 100) np.testing.assert_array_equal(data_mat_res1_iso, data_mat_res1_aniso) # Get data at res 4 with iso flag and verify it's non-zero and DIFFERENT than without flag request = factory.get( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/4/0:256/0:256/0:8/?iso=false', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='4', x_range='0:256', y_range='0:256', z_range='0:8', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) raw_data = blosc.decompress(response.content) data_mat_res4_aniso = np.fromstring(raw_data, dtype=np.uint8) data_mat_res4_aniso = np.reshape(data_mat_res4_aniso, (8, 256, 256), order='C') # Make sure not blank self.assertGreater(data_mat_res4_aniso.sum(), 1) # Get data at res 4 with iso flag and verify it's non-zero and DIFFERENT than without flag request = factory.get( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/4/0:256/0:256/0:8/?iso=true', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='4', x_range='0:256', y_range='0:256', z_range='0:8', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) raw_data = blosc.decompress(response.content) data_mat_res4_iso = np.fromstring(raw_data, dtype=np.uint8) data_mat_res4_iso = np.reshape(data_mat_res4_iso, (8, 256, 256), order='C') # Make sure not blank self.assertGreater(data_mat_res4_iso.sum(), 1) self.assertRaises(AssertionError, np.testing.assert_array_equal, data_mat_res4_aniso, data_mat_res4_iso) # Post data, invalidating the downsample operation request = factory.post( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/0/0:1024/0:1024/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='0', x_range='0:1024', y_range='0:1024', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Verify now NOT downsampled request = factory.get('/' + version + '/downsample/col1/exp_ds_aniso/channel1/', content_type='application/json') # log in user force_authenticate(request, user=self.user) response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1').render() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["num_hierarchy_levels"], 5) self.assertEqual(response.data["status"], "NOT_DOWNSAMPLED")
def setUpClass(cls): """ get_some_resource() is slow, to avoid calling it for each test use setUpClass() and store the result as class variable """ super(ImageViewIntegrationTests, cls).setUpClass() # Setup the helper to create temporary AWS resources cls.setup_helper = SetupTests() cls.setup_helper.mock = False # Create a user in django dbsetup = SetupTestDB() cls.user = dbsetup.create_user('testuser') dbsetup.add_role('resource-manager') dbsetup.set_user(cls.user) # Populate django models DB dbsetup.insert_spatialdb_test_data() try: cls.setup_helper.create_s3_index_table( OBJECTIO_CONFIG["s3_index_table"]) except ClientError: cls.setup_helper.delete_s3_index_table( OBJECTIO_CONFIG["s3_index_table"]) cls.setup_helper.create_s3_index_table( OBJECTIO_CONFIG["s3_index_table"]) try: cls.setup_helper.create_cuboid_bucket( OBJECTIO_CONFIG["cuboid_bucket"]) except ClientError: cls.setup_helper.delete_cuboid_bucket( OBJECTIO_CONFIG["cuboid_bucket"]) cls.setup_helper.create_cuboid_bucket( OBJECTIO_CONFIG["cuboid_bucket"]) try: OBJECTIO_CONFIG[ "s3_flush_queue"] = cls.setup_helper.create_flush_queue( FLUSH_QUEUE_NAME) except ClientError: try: cls.setup_helper.delete_flush_queue( OBJECTIO_CONFIG["s3_flush_queue"]) except: pass time.sleep(61) OBJECTIO_CONFIG[ "s3_flush_queue"] = cls.setup_helper.create_flush_queue( FLUSH_QUEUE_NAME) # load some data for reading test_mat = np.random.randint(1, 254, (16, 1024, 1024)) cls.test_data_8 = test_mat.astype(np.uint8) h = cls.test_data_8.tobytes() bb = blosc.compress(h, typesize=8) # Post data to the database factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel1/0/0:1024/0:1024/0:16/', bb, content_type='application/blosc') force_authenticate(request, user=cls.user) _ = Cutout.as_view()(request, collection='col1', experiment='exp1', dataset='channel1', resolution='0', x_range='0:1024', y_range='0:1024', z_range='0:16')