def test_request_cutout_invalid_zargs(self): """ Test initialization of cutout arguments for a invalid cutout request. The x-args are outside the coordinate frame :return: """ url = '/' + version + '/cutout/col1/exp1/channel1/0/0:6/0:6/0:1040/' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "collection_name": "col1", "experiment_name": "exp1", "channel_name": "channel1", "resolution": 2, "x_args": "0:6", "y_args": "0:6", "z_args": "0:1040", "time_args": None } with self.assertRaises(BossError): BossRequest(drfrequest, request_args)
def test_request_cutout_filter_no_time(self): """ Test initialization of boss_key for a time sample range :return: """ url = '/' + version + '/cutout/col1/exp1/channel3/2/0:5/0:6/0:2/?filter=1,2,3,4' expected_ids = np.array([1, 2, 3, 4]) expected_ids = expected_ids.astype(np.uint64) # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": 'col1', "experiment_name": 'exp1', "channel_name": 'channel3', "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None, "ids": '1,2,3,4' } ret = BossRequest(drfrequest, request_args) self.assertEqual(np.array_equal(ret.get_filter_ids(),expected_ids),True)
def test_basic_resource_get_downsampled_extent_dims_isotropic(self): """Test downsample extent isotropic Returns: None """ url = '/' + version + '/cutout/col1/exp_iso/channel1/0/0:5/0:6/0:2/' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version request_args = { "service": "cutout", "collection_name": "col1", "experiment_name": "exp_iso", "channel_name": "channel1", "resolution": 0, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } req = BossRequest(drfrequest, request_args) resource = BossResourceDjango(req) extent_dims = resource.get_downsampled_extent_dims() self.assertEqual(len(extent_dims), 8) self.assertEqual(extent_dims[0], [2000, 5000, 200]) self.assertEqual(extent_dims[1], [1000, 2500, 100]) self.assertEqual(extent_dims[3], [250, 625, 25])
def test_request_cutout_init_cutoutargs_time_range(self): """ Test initialization of boss_key for a time sample range :return: """ url = '/' + version + '/cutout/col1/exp1/channel1/2/0:5/0:6/0:2/1:5/' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": 'col1', "experiment_name": 'exp1', "channel_name": 'channel1', "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": "1:5" } ret = BossRequest(drfrequest, request_args) time = ret.get_time() self.assertEqual(time, range(1, 5)) self.assertEqual(ret.time_request, True)
def test_basic_resource_get_iso_level_isotropic(self): """Test get iso level isotropic Returns: None """ url = '/' + version + '/cutout/col1/exp_iso/channel1/0/0:5/0:6/0:2/' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version request_args = { "service": "cutout", "collection_name": "col1", "experiment_name": "exp_iso", "channel_name": "channel1", "resolution": 0, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } req = BossRequest(drfrequest, request_args) resource = BossResourceDjango(req) self.assertEqual(resource.get_isotropic_level(), 0)
def test_request_cutout_valid_base_resolution(self): """ Test cutout request when channel has a non-zero base resolution """ col = 'col1' channel = CHAN_BASE_RES res = NUM_HIERARCHY_LEVELS + BASE_RESOLUTION - 1 url = '/{}/cutout/{}/{}/{}/{}/0:5/0:6/0:2/'.format( version, col, EXP_BASE_RES, channel, res) # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": col, "experiment_name": EXP_BASE_RES, "channel_name": channel, "resolution": res, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } # No exception should be raised. BossRequest(drfrequest, request_args)
def test_request_cutout_lookupkey(self): """ Test initialization of boss_key for a time sample range :return: """ url = '/' + version + '/cutout/col1/exp1/channel1/2/0:5/0:6/0:2/1:5/' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": 'col1', "experiment_name": 'exp1', "channel_name": 'channel1', "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": "1:5" } ret = BossRequest(drfrequest, request_args) col_id = ret.collection.pk exp_id = ret.experiment.pk channel_id = ret.channel.pk base_lookup = str(col_id) + '&' + str(exp_id) + '&' + str(channel_id) self.assertEqual(base_lookup, ret.get_lookup_key())
def test_request_cutout_invalid_resolution(self): """ Test initialization of cutout requests for an invalid datamodel - experiment does not exist for the collection :return: """ url = '/' + version + '/cutout/col2/exp1/channel1/92/0:5/0:6/0:2/' col = 'col1' exp = 'exp1' channel = 'channel1' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": col, "experiment_name": exp, "channel_name": channel, "resolution": 92, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } with self.assertRaises(BossError): BossRequest(drfrequest, request_args)
def test_request_cutout_filter_no_time(self): """ Test initialization of boss_key for a time sample range :return: """ url = '/' + version + '/cutout/col1/exp1/channel3/2/0:5/0:6/0:2/?filter=1,2,3,4' expected_ids = np.array([1, 2, 3, 4]) expected_ids = expected_ids.astype(np.uint64) # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": 'col1', "experiment_name": 'exp1', "channel_name": 'channel3', "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None, "ids": '1,2,3,4' } ret = BossRequest(drfrequest, request_args) self.assertEqual(np.array_equal(ret.get_filter_ids(), expected_ids), True)
def test_request_cutout_invalid_resolution_past_upper_bound(self): """ Test initialization of cutout requests with an invalid resolution """ col = 'col1' channel = 'channel1' res = NUM_HIERARCHY_LEVELS url = '/{}/cutout/{}/{}/{}/{}/0:5/0:6/0:2/'.format( version, col, EXP1, channel, res) # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": col, "experiment_name": EXP1, "channel_name": channel, "resolution": res, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } with self.assertRaises(BossError): BossRequest(drfrequest, request_args)
def test_request_cutout_invalid_datamodel(self): """ Test initialization of cutout requests for an invalid datamodel - experiment does not exist for the collection :return: """ url = '/' + version + '/cutout/col2/exp1/channel1/2/0:5/0:6/0:2/' col = 'col1' exp = 'exp1' channel = 'channel12345' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": col, "experiment_name": exp, "channel_name": channel, "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } with self.assertRaises(BossError): BossRequest(drfrequest, request_args)
def test_request_cutout_invalid_base_resolution_above_upper_bound(self): """ Test cutout request when channel has a non-zero base resolution - pass resolution equal to the base resolution + num hierarchy levels """ col = 'col1' channel = CHAN_BASE_RES res = BASE_RESOLUTION + NUM_HIERARCHY_LEVELS url = '/{}/cutout/{}/{}/{}/{}/0:5/0:6/0:2/'.format( version, col, EXP_BASE_RES, channel, res) # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": col, "experiment_name": EXP_BASE_RES, "channel_name": channel, "resolution": res, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } with self.assertRaises(BossError): BossRequest(drfrequest, request_args)
def test_channel_uint16_cuboid_unaligned_offset_time_blosc(self): """ Test uint16 data, not cuboid aligned, offset, time samples, blosc interface Test Requires >=2GB of memory! """ test_mat = np.random.randint(1, 2**16 - 1, (3, 17, 300, 500)) test_mat = test_mat.astype(np.uint16) h = test_mat.tobytes() bb = blosc.compress(h, typesize=16) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel2/0/100:600/450:750/20:37/0:3', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', dataset='channel2', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get( '/' + version + '/cutout/col1/exp1/channel2/0/100:600/450:750/20:37/0:3', HTTP_ACCEPT='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', dataset='channel2', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint16) data_mat = np.reshape(data_mat, (3, 17, 300, 500), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_get_object_bounding_box_single_cuboid(self): """ Test getting the bounding box of a object""" test_mat = np.ones((128, 128, 16)) test_mat[0:128, 0:128, 0:16] = 4 test_mat = test_mat.astype(np.uint64) test_mat = test_mat.reshape((16, 128, 128)) bb = blosc.compress(test_mat, typesize=64) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/bbchan1/0/1536:1664/1536:1664/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='bbchan1', resolution='0', x_range='1536:1664', y_range='1536:1664', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/bbchan1/0/1536:1664/1536:1664/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='bbchan1', resolution='0', x_range='1536:1664', y_range='1536:1664', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint64) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat) # get the bounding box # Create request factory = APIRequestFactory() request = factory.get('/' + version + '/boundingbox/col1/exp1/bbchan1/0/4') # log in user force_authenticate(request, user=self.user) # Make request response = BoundingBox.as_view()(request, collection='col1', experiment='exp1', channel='bbchan1', resolution='0', id='4') self.assertEqual(response.status_code, status.HTTP_200_OK) bb = response.data self.assertEqual(bb['t_range'], [0, 1]) self.assertEqual(bb['x_range'], [1536, 2048]) self.assertEqual(bb['y_range'], [1536, 2048]) self.assertEqual(bb['z_range'], [0, 16])
def test_channel_uint16_cuboid_aligned_offset_no_time_blosc(self): """ Test uint16 data, cuboid aligned, offset, no time samples, blosc interface""" test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint16) h = test_mat.tobytes() bb = blosc.compress(h, typesize=16) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get( '/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint16) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint64_cuboid_unaligned_offset_time_offset_blosc_numpy( self): """ Test uint64 data, not cuboid aligned, offset, time samples, blosc interface Test Requires >=2GB of memory! """ test_mat = np.random.randint(1, 2**50, (3, 17, 300, 500)) test_mat = test_mat.astype(np.uint64) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', dataset='layer1', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get( '/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203', HTTP_ACCEPT='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', dataset='layer1', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_mat = blosc.unpack_array(response.content) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint64_cuboid_aligned_no_offset_no_time_blosc(self): """ Test uint64 data, cuboid aligned, no offset, no time samples""" test_mat = np.ones((128, 128, 16)) test_mat = test_mat.astype(np.uint64) test_mat = test_mat.reshape((16, 128, 128)) bb = blosc.compress(test_mat, typesize=64) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint64) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat) # get the bounding box # Create request factory = APIRequestFactory() request = factory.get('/' + version + '/ids/col1/exp1/layer1/0/0:128/0:128/0:16/') # log in user force_authenticate(request, user=self.user) # Make request response = Ids.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data['ids'], ['1'])
def test_channel_uint64_time_npygz_upload(self): """ Test uint8 data, using the npygz interface with time series support while uploading in that format as well """ test_mat = np.random.randint(1, 256, (3, 4, 128, 128)) test_mat = test_mat.astype(np.uint64) # Save Data to npy npy_file = io.BytesIO() np.save(npy_file, test_mat, allow_pickle=False) # Compress npy npy_gz = zlib.compress(npy_file.getvalue()) # Send file npy_gz_file = io.BytesIO(npy_gz) npy_gz_file.seek(0) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13', npy_gz_file.read(), content_type='application/npygz') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:4', t_range='10:13') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13', HTTP_ACCEPT='application/npygz') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:4', t_range='10:13').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_bytes = zlib.decompress(response.content) # Open data_obj = io.BytesIO(data_bytes) data_mat = np.load(data_obj) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint16_cuboid_aligned_no_offset_no_time_blosc_numpy(self): """ Test uint16 data, cuboid aligned, no offset, no time samples""" test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint16) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel2/0/0:128/0:128/0:16/', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel2/0/0:128/0:128/0:16/', HTTP_ACCEPT='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_mat = blosc.unpack_array(response.content) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def setUp(self): """ Copy params from the Layer setUpClass """ # Setup config self.kvio_config = self.layer.kvio_config self.state_config = self.layer.state_config self.object_store_config = self.layer.object_store_config self.user = self.layer.user # Log Django User in self.client.force_login(self.user) if not self.test_data_loaded: # Flush cache between tests client = redis.StrictRedis(host=self.kvio_config['cache_host'], port=6379, db=1, decode_responses=False) client.flushdb() client = redis.StrictRedis(host=self.state_config['cache_state_host'], port=6379, db=1, decode_responses=False) client.flushdb() # load some data for reading self.test_data_8 = np.random.randint(1, 254, (16, 1024, 1024), dtype=np.uint8) bb = blosc.compress(self.test_data_8, typesize=8) # Post data to the database factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel1/0/0:1024/0:1024/0:16/', bb, content_type='application/blosc') force_authenticate(request, user=self.user) _ = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:1024', y_range='0:1024', z_range='0:16', t_range=None) self.test_data_loaded = True
def test_channel_uint16_wrong_data_type(self): """ Test posting the wrong bitdepth data """ test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint8) h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel2/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_channel_uint16_wrong_dimensions_numpy(self): """ Test posting with the wrong xyz dims using the numpy interface""" test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint16) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel2/0/0:100/0:128/0:16/', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:100', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_channel_uint16_get_too_big(self): """ Test getting a cutout that is over 1GB uncompressed""" # Create request factory = APIRequestFactory() # Create Request to get data you posted request = factory.get( '/' + version + '/cutout/col1/exp1/channel2/0/0:100000/0:100000/0:10000/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:100000', y_range='0:100000', z_range='0:10000', t_range=None) self.assertEqual(response.status_code, status.HTTP_413_REQUEST_ENTITY_TOO_LARGE)
def test_request_cutout_invalid_deleted_channel(self): """ Test initialization of cutout requests for channel that was just deleted """ # Post a new channel url = '/' + version + '/collection/col1/experiment/exp1/channel/channel33/' data = { 'description': 'This is a new channel', 'type': 'annotation', 'datatype': 'uint64', 'sources': ['channel1'], 'related': ['channel2'] } response = self.client.post(url, data=data) self.assertEqual(response.status_code, 201) url = '/' + version + '/collection/col1/experiment/exp1/channel/channel33' response = self.client.delete(url) self.assertEqual(response.status_code, 204) url = '/' + version + '/cutout/col2/exp1/channel33/0/0:5/0:6/0:2/' col = 'col1' exp = 'exp1' channel = 'channel33' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": col, "experiment_name": exp, "channel_name": channel, "resolution": 0, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } with self.assertRaises(BossError): BossRequest(drfrequest, request_args)
def test_request_cutout_init_cutoutargs_channel(self): """ Test initialization of cutout arguments for a cutout request :return: """ url = '/' + version + '/cutout/col1/exp1/channel1/2/0:5/0:6/0:2/' res = 2 (x_start, x_stop) = (0, 5) (y_start, y_stop) = (0, 6) (z_start, z_stop) = (0, 2) # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": 'col1', "experiment_name": 'exp1', "channel_name": 'channel1', "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } ret = BossRequest(drfrequest, request_args) self.assertEqual(ret.get_resolution(), res) self.assertEqual(ret.get_x_start(), x_start) self.assertEqual(ret.get_x_stop(), x_stop) self.assertEqual(ret.get_x_span(), x_stop - x_start) self.assertEqual(ret.get_y_start(), y_start) self.assertEqual(ret.get_y_stop(), y_stop) self.assertEqual(ret.get_y_span(), y_stop - y_start) self.assertEqual(ret.get_z_start(), z_start) self.assertEqual(ret.get_z_stop(), z_stop) self.assertEqual(ret.get_z_span(), z_stop - z_start)
def test_channel_uint64_filter_ids_not_found(self): """ Test filter_cutout by ids not in the region""" test_mat = np.ones((128, 128, 4)) test_mat[0][0][0] = 2 test_mat[0][0][1] = 3 test_mat[0][0][2] = 4 test_mat = test_mat.reshape(4, 128, 128) test_mat = test_mat.astype(np.uint64) h = test_mat.tobytes() bb = blosc.compress(h, typesize=64) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/128:256/256:384/16:20/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='128:256', y_range='256:384', z_range='16:20', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/128:256/256:384/16:20/?filter=5,6,7', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='128:256', y_range='256:384', z_range='16:20', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint64) data_mat = np.reshape(data_mat, (4, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(np.unique(data_mat), np.arange(0, 1, dtype=np.uint64))
def test_channel_uint64_time_npygz_download(self): """ Test uint8 data, using the npygz interface with time series support """ test_mat = np.random.randint(1, 256, (3, 4, 128, 128)) test_mat = test_mat.astype(np.uint64) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:4', t_range='10:13') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13', HTTP_ACCEPT='application/npygz') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:4', t_range='10:13').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_bytes = zlib.decompress(response.content) # Open data_obj = io.BytesIO(data_bytes) data_mat = np.load(data_obj) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint16_cuboid_unaligned_offset_time_blosc(self): """ Test uint16 data, not cuboid aligned, offset, time samples, blosc interface Test Requires >=2GB of memory! """ test_mat = np.random.randint(1, 2**16-1, (3, 17, 300, 500)) test_mat = test_mat.astype(np.uint16) h = test_mat.tobytes() bb = blosc.compress(h, typesize=16) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/100:600/450:750/20:37/0:3', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='0:3') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel2/0/100:600/450:750/20:37/0:3', HTTP_ACCEPT='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='0:3').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint16) data_mat = np.reshape(data_mat, (3, 17, 300, 500), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_request_cutout_invalid_deleted_channel(self): """ Test initialization of cutout requests for channel that was just deleted """ # Post a new channel url = '/' + version + '/collection/col1/experiment/exp1/channel/channel33/' data = {'description': 'This is a new channel', 'type': 'annotation', 'datatype': 'uint64', 'sources': ['channel1'], 'related': ['channel2']} response = self.client.post(url, data=data) self.assertEqual(response.status_code, 201) url = '/' + version + '/collection/col1/experiment/exp1/channel/channel33' response = self.client.delete(url) self.assertEqual(response.status_code, 204) url = '/' + version + '/cutout/col2/exp1/channel33/0/0:5/0:6/0:2/' col = 'col1' exp = 'exp1' channel = 'channel33' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": col, "experiment_name": exp, "channel_name": channel, "resolution": 0, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } with self.assertRaises(BossError): BossRequest(drfrequest, request_args)
def test_channel_uint64_cuboid_aligned_offset_no_time_blosc(self): """ Test uint64 data, cuboid aligned, offset, no time samples, blosc interface""" test_mat = np.random.randint(1, 256, (4, 128, 128)) test_mat = test_mat.astype(np.uint64) h = test_mat.tobytes() bb = blosc.compress(h, typesize=64) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/128:256/256:384/16:20/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='128:256', y_range='256:384', z_range='16:20', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/128:256/256:384/16:20/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='128:256', y_range='256:384', z_range='16:20', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint64) data_mat = np.reshape(data_mat, (4, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_channel_uint64_cuboid_unaligned_offset_time_offset_blosc_numpy(self): """ Test uint64 data, not cuboid aligned, offset, time samples, blosc interface Test Requires >=2GB of memory! """ test_mat = np.random.randint(1, 256, (3, 17, 300, 500)) test_mat = test_mat.astype(np.uint64) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='200:203') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203', HTTP_ACCEPT='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1', resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='200:203').render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_mat = blosc.unpack_array(response.content) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_basic_resource_get_downsampled_voxel_dims_anisotropic_iso(self): """Test downsample voxel dims anisotropic with iso flag Returns: None """ url = '/' + version + '/cutout/col1/exp_aniso/channel1/0/0:5/0:6/0:2/' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version request_args = { "service": "cutout", "collection_name": "col1", "experiment_name": "exp_aniso", "channel_name": "channel1", "resolution": 0, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } req = BossRequest(drfrequest, request_args) resource = BossResourceDjango(req) voxel_dims = resource.get_downsampled_voxel_dims(iso=True) self.assertEqual(len(voxel_dims), 8) self.assertEqual(voxel_dims[0], [4, 4, 35]) self.assertEqual(voxel_dims[1], [8, 8, 35]) self.assertEqual(voxel_dims[2], [16, 16, 35]) self.assertEqual(voxel_dims[3], [32, 32, 35]) self.assertEqual(voxel_dims[4], [64, 64, 70]) self.assertEqual(voxel_dims[5], [128, 128, 140])
def test_request_cutout_init_channel(self): """ Test initialization of cutout requests for the datamodel :return: """ url = '/' + version + '/cutout/col1/exp1/channel1/2/0:5/0:6/0:2/' col = 'col1' exp = 'exp1' channel = 'channel1' boss_key = 'col1&exp1&channel1' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "version": version, "collection_name": col, "experiment_name": exp, "channel_name": channel, "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } ret = BossRequest(drfrequest, request_args) self.assertEqual(ret.get_collection(), col) self.assertEqual(ret.get_experiment(), exp) self.assertEqual(ret.get_channel(), channel) self.assertEqual(ret.get_boss_key(), boss_key) self.assertEqual(ret.time_request, False)
def test_channel_uint16_cuboid_aligned_offset_no_time_blosc_numpy(self): """ Test uint16 data, cuboid aligned, offset, no time samples, blosc interface""" test_mat = np.random.randint(1, 2**16-1, (16, 128, 128)) test_mat = test_mat.astype(np.uint16) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/', HTTP_ACCEPT='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress data_mat = blosc.unpack_array(response.content) # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def setUp(self): """Setup test by inserting data model items into the database""" self.rf = APIRequestFactory() dbsetup = SetupTestDB() self.user = dbsetup.create_user() dbsetup.add_role("resource-manager", self.user) self.client.force_login(self.user) dbsetup.insert_cloudvolume_test_data() url = '/' + version + '/cutout/col1/exp1/chan1/2/0:5/0:6/0:2/' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "collection_name": "col1", "experiment_name": "exp1", "channel_name": "chan1", "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } self.request_channel = BossRequest(drfrequest, request_args) # Setup Layer url = '/' + version + '/cutout/col1/exp1/anno1/2/0:5/0:6/0:2/' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "collection_name": "col1", "experiment_name": "exp1", "channel_name": "anno1", "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } self.request_annotation = BossRequest(drfrequest, request_args)
def test_channel_uint16_get_too_big(self): """ Test getting a cutout that is over 1GB uncompressed""" # Create request factory = APIRequestFactory() # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel2/0/0:100000/0:100000/0:10000/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:100000', y_range='0:100000', z_range='0:10000', t_range=None) self.assertEqual(response.status_code, status.HTTP_413_REQUEST_ENTITY_TOO_LARGE)
def test_channel_uint16_wrong_data_type_numpy(self): """ Test posting the wrong bitdepth data using the blosc-numpy interface""" test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint8) bb = blosc.pack_array(test_mat) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/0:128/0:128/0:16/', bb, content_type='application/blosc-python') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_channel_uint16_wrong_dimensions(self): """ Test posting with the wrong xyz dims""" test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128)) test_mat = test_mat.astype(np.uint16) h = test_mat.tobytes() bb = blosc.compress(h, typesize=16) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/0:100/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2', resolution='0', x_range='0:100', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_channel_uint8_cuboid_aligned_offset(self): "uint8 - cuboid aligned - offset present" # Get test data from cloudvolume, returned in XYZT test_mat = self.vol_uint8[0:512, 0:512, 16:32] # Convert to ZYX test_mat = np.squeeze(test_mat).T # Create Request to get data you posted factory = APIRequestFactory() request = factory.get( '/' + version + '/cutout/col1/exp1/chan1/0/128:256/256:384/16:32/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='chan1', resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint8) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) offset_mat = test_mat[:, 256:384, 128:256] np.testing.assert_array_equal(data_mat, offset_mat)
def test_delayed_write(self): """A test with multiple writes to the same cuboid""" test_mat = np.random.randint(1, 254, (16, 128, 128)) test_mat = test_mat.astype(np.uint8) h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) # Create request factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) test_mat[1, 20, 40] = 5 test_mat[1, 20, 41] = 5 test_mat[1, 20, 42] = 5 h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint8) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def setUp(self): """Setup test by inserting data model items into the database""" self.rf = APIRequestFactory() dbsetup = SetupTestDB() self.user = dbsetup.create_user() dbsetup.add_role("resource-manager", self.user) self.client.force_login(self.user) dbsetup.insert_test_data() dbsetup.insert_iso_data() url = '/' + version + '/collection/col1/experiment/exp1/channel/channel33/' data = {'description': 'This is a new channel', 'type': 'annotation', 'datatype': 'uint8', 'sources': ['channel1'], 'related': ['channel2', 'channel3']} response = self.client.post(url, data=data) url = '/' + version + '/cutout/col1/exp1/channel1/2/0:5/0:6/0:2/' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "collection_name": "col1", "experiment_name": "exp1", "channel_name": "channel1", "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } self.request_channel = BossRequest(drfrequest, request_args) # Setup Layer url = '/' + version + '/cutout/col1/exp1/layer1/2/0:5/0:6/0:2/' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "collection_name": "col1", "experiment_name": "exp1", "channel_name": "layer1", "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } self.request_annotation = BossRequest(drfrequest, request_args) url = '/' + version + '/cutout/col1/exp1/channel33/2/0:5/0:6/0:2/' # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Cutout().initialize_request(request) drfrequest.version = version # Create the request dict request_args = { "service": "cutout", "collection_name": "col1", "experiment_name": "exp1", "channel_name": "channel33", "resolution": 2, "x_args": "0:5", "y_args": "0:6", "z_args": "0:2", "time_args": None } self.request_channel_links = BossRequest(drfrequest, request_args)
def test_delayed_write(self): """A test with multiple writes to the same cuboid""" test_mat = np.random.randint(1, 254, (16, 128, 128)) test_mat = test_mat.astype(np.uint8) h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) # Create request factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) test_mat[1, 20, 40] = 5 test_mat[1, 20, 41] = 5 test_mat[1, 20, 42] = 5 h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) factory = APIRequestFactory() request = factory.post('/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Create Request to get data you posted request = factory.get('/' + version + '/cutout/col1/exp1/channel1/0/0:128/0:128/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel1', resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Decompress raw_data = blosc.decompress(response.content) data_mat = np.fromstring(raw_data, dtype=np.uint8) data_mat = np.reshape(data_mat, (16, 128, 128), order='C') # Test for data equality (what you put in is what you got back!) np.testing.assert_array_equal(data_mat, test_mat)
def test_start_downsample_get_status_and_check_data(self): """A large complex test that verifies all the pluming for downsample. Does not validate data integrity, but does make sure data exists at different levels and iso vs. aniso.""" self.dbsetup.insert_downsample_data() # Post some data to the channel test_mat = np.random.randint(1, 254, (16, 1024, 1024)) test_mat = test_mat.astype(np.uint8) h = test_mat.tobytes() bb = blosc.compress(h, typesize=8) factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/0/0:1024/0:1024/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='0', x_range='0:1024', y_range='0:1024', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Wait for data to be written request = factory.get( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/0/0:1024/0:1024/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='0', x_range='0:1024', y_range='0:1024', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) # Trigger downsample factory = APIRequestFactory() request = factory.post('/' + version + '/downsample/col1/exp_ds_aniso/channel1/', content_type='application/json') # log in user force_authenticate(request, user=self.user) # Make request response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1') self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Make Sure status has changed factory = APIRequestFactory() request = factory.get('/' + version + '/downsample/col1/exp_ds_aniso/channel1/', content_type='application/json') # log in user force_authenticate(request, user=self.user) # Make request response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1').render() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["num_hierarchy_levels"], 5) self.assertEqual(response.data["status"], "IN_PROGRESS") for _ in range(0, 30): # Make request response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1').render() self.assertEqual(response.status_code, status.HTTP_200_OK) if response.data["status"] != "IN_PROGRESS": break time.sleep(2) # Verify now downsampled response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1').render() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["num_hierarchy_levels"], 5) self.assertEqual(response.data["status"], "DOWNSAMPLED") # Get data at res 1 and verify it's non-zero request = factory.get( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/1/0:512/0:512/0:16/', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='1', x_range='0:512', y_range='0:512', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) raw_data = blosc.decompress(response.content) data_mat_res1_aniso = np.fromstring(raw_data, dtype=np.uint8) data_mat_res1_aniso = np.reshape(data_mat_res1_aniso, (16, 512, 512), order='C') # Make sure not blank self.assertGreater(data_mat_res1_aniso.sum(), 100) # Get data at res 1 with iso flag and verify it's non-zero and the same as without flag request = factory.get( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/1/0:512/0:512/0:16/?iso=true', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='1', x_range='0:512', y_range='0:512', z_range='0:16', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) raw_data = blosc.decompress(response.content) data_mat_res1_iso = np.fromstring(raw_data, dtype=np.uint8) data_mat_res1_iso = np.reshape(data_mat_res1_iso, (16, 512, 512), order='C') # Make sure not blank self.assertGreater(data_mat_res1_iso.sum(), 100) np.testing.assert_array_equal(data_mat_res1_iso, data_mat_res1_aniso) # Get data at res 4 with iso flag and verify it's non-zero and DIFFERENT than without flag request = factory.get( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/4/0:256/0:256/0:8/?iso=false', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='4', x_range='0:256', y_range='0:256', z_range='0:8', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) raw_data = blosc.decompress(response.content) data_mat_res4_aniso = np.fromstring(raw_data, dtype=np.uint8) data_mat_res4_aniso = np.reshape(data_mat_res4_aniso, (8, 256, 256), order='C') # Make sure not blank self.assertGreater(data_mat_res4_aniso.sum(), 1) # Get data at res 4 with iso flag and verify it's non-zero and DIFFERENT than without flag request = factory.get( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/4/0:256/0:256/0:8/?iso=true', accepts='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='4', x_range='0:256', y_range='0:256', z_range='0:8', t_range=None).render() self.assertEqual(response.status_code, status.HTTP_200_OK) raw_data = blosc.decompress(response.content) data_mat_res4_iso = np.fromstring(raw_data, dtype=np.uint8) data_mat_res4_iso = np.reshape(data_mat_res4_iso, (8, 256, 256), order='C') # Make sure not blank self.assertGreater(data_mat_res4_iso.sum(), 1) self.assertRaises(AssertionError, np.testing.assert_array_equal, data_mat_res4_aniso, data_mat_res4_iso) # Post data, invalidating the downsample operation request = factory.post( '/' + version + '/cutout/col1/exp_ds_aniso/channel1/0/0:1024/0:1024/0:16/', bb, content_type='application/blosc') # log in user force_authenticate(request, user=self.user) # Make request response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1', resolution='0', x_range='0:1024', y_range='0:1024', z_range='0:16', t_range=None) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # Verify now NOT downsampled request = factory.get('/' + version + '/downsample/col1/exp_ds_aniso/channel1/', content_type='application/json') # log in user force_authenticate(request, user=self.user) response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1').render() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["num_hierarchy_levels"], 5) self.assertEqual(response.data["status"], "NOT_DOWNSAMPLED")
def setUpClass(cls): """ get_some_resource() is slow, to avoid calling it for each test use setUpClass() and store the result as class variable """ super(ImageViewIntegrationTests, cls).setUpClass() # Setup the helper to create temporary AWS resources cls.setup_helper = SetupTests() cls.setup_helper.mock = False # Create a user in django dbsetup = SetupTestDB() cls.user = dbsetup.create_user('testuser') dbsetup.add_role('resource-manager') dbsetup.set_user(cls.user) # Populate django models DB dbsetup.insert_spatialdb_test_data() try: cls.setup_helper.create_s3_index_table( OBJECTIO_CONFIG["s3_index_table"]) except ClientError: cls.setup_helper.delete_s3_index_table( OBJECTIO_CONFIG["s3_index_table"]) cls.setup_helper.create_s3_index_table( OBJECTIO_CONFIG["s3_index_table"]) try: cls.setup_helper.create_cuboid_bucket( OBJECTIO_CONFIG["cuboid_bucket"]) except ClientError: cls.setup_helper.delete_cuboid_bucket( OBJECTIO_CONFIG["cuboid_bucket"]) cls.setup_helper.create_cuboid_bucket( OBJECTIO_CONFIG["cuboid_bucket"]) try: OBJECTIO_CONFIG[ "s3_flush_queue"] = cls.setup_helper.create_flush_queue( FLUSH_QUEUE_NAME) except ClientError: try: cls.setup_helper.delete_flush_queue( OBJECTIO_CONFIG["s3_flush_queue"]) except: pass time.sleep(61) OBJECTIO_CONFIG[ "s3_flush_queue"] = cls.setup_helper.create_flush_queue( FLUSH_QUEUE_NAME) # load some data for reading test_mat = np.random.randint(1, 254, (16, 1024, 1024)) cls.test_data_8 = test_mat.astype(np.uint8) h = cls.test_data_8.tobytes() bb = blosc.compress(h, typesize=8) # Post data to the database factory = APIRequestFactory() request = factory.post( '/' + version + '/cutout/col1/exp1/channel1/0/0:1024/0:1024/0:16/', bb, content_type='application/blosc') force_authenticate(request, user=cls.user) _ = Cutout.as_view()(request, collection='col1', experiment='exp1', dataset='channel1', resolution='0', x_range='0:1024', y_range='0:1024', z_range='0:16')