def test_get_iso_properties_iso_false(self):
""" Test getting the properties of an isotropic channel with arg but false"""
# Create request
factory = APIRequestFactory()
request = factory.get('/' + version +
'/downsample/col1/exp_iso/channel1/?iso=False',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request,
collection='col1',
experiment='exp_iso',
channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["num_hierarchy_levels"], 8)
self.assertEqual(response.data["status"], "NOT_DOWNSAMPLED")
self.assertEqual(response.data["voxel_size"]['0'], [6.0, 6.0, 6.0])
self.assertEqual(response.data["voxel_size"]['3'], [48.0, 48.0, 48.0])
self.assertEqual(response.data["voxel_size"]['5'],
[192.0, 192.0, 192.0])
self.assertEqual(response.data["extent"]['0'], [2000, 5000, 200])
self.assertEqual(response.data["extent"]['3'], [250, 625, 25])
self.assertEqual(response.data["extent"]['5'], [63, 157, 7])
self.assertEqual(response.data["cuboid_size"]['0'], [512, 512, 16])
self.assertEqual(response.data["cuboid_size"]['3'], [512, 512, 16])
self.assertEqual(response.data["cuboid_size"]['5'], [512, 512, 16])
def test_get_iso_properties_iso(self):
""" Test getting the properties of an isotropic channel with arg but true"""
# Create request
factory = APIRequestFactory()
request = factory.get('/' + version + '/downsample/col1/exp_iso/channel1/?iso=True',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request, collection='col1', experiment='exp_iso', channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["num_hierarchy_levels"], 8)
self.assertEqual(response.data["status"], "NOT_DOWNSAMPLED")
self.assertEqual(response.data["voxel_size"]['0'], [6.0, 6.0, 6.0])
self.assertEqual(response.data["voxel_size"]['3'], [48.0, 48.0, 48.0])
self.assertEqual(response.data["voxel_size"]['5'], [192.0, 192.0, 192.0])
self.assertEqual(response.data["extent"]['0'], [2000, 5000, 200])
self.assertEqual(response.data["extent"]['3'], [250, 625, 25])
self.assertEqual(response.data["extent"]['5'], [63, 157, 7])
self.assertEqual(response.data["cuboid_size"]['0'], [512, 512, 16])
self.assertEqual(response.data["cuboid_size"]['3'], [512, 512, 16])
self.assertEqual(response.data["cuboid_size"]['5'], [512, 512, 16])
def test_start_and_cancel_downsample_aniso(self):
self.dbsetup.insert_downsample_data()
factory = APIRequestFactory()
request = factory.post('/' + version + '/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso',
channel='channel1')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# Make Sure status has changed
factory = APIRequestFactory()
request = factory.get('/' + version + '/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso',
channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["num_hierarchy_levels"], 5)
self.assertEqual(response.data["status"], "IN_PROGRESS")
# Cancel the downsample job
request = factory.delete('/' + version + '/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso',
channel='channel1')
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
# Make Sure status has changed
factory = APIRequestFactory()
request = factory.get('/' + version + '/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso',
channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], "NOT_DOWNSAMPLED")
# Try to cancel the downsample job again, but it won't because in NOT_DOWNSAMPLED state
request = factory.delete('/' + version + '/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso',
channel='channel1')
self.assertEqual(response.status_code, status.HTTP_409_CONFLICT)
def test_start_downsample_get_status_and_check_data(self):
"""A large complex test that verifies all the pluming for downsample.
Does not validate data integrity, but does make sure data exists at different levels and iso vs. aniso."""
self.dbsetup.insert_downsample_data()
# Post some data to the channel
test_mat = np.random.randint(1, 254, (16, 1024, 1024))
test_mat = test_mat.astype(np.uint8)
h = test_mat.tobytes()
bb = blosc.compress(h, typesize=8)
factory = APIRequestFactory()
request = factory.post(
'/' + version +
'/cutout/col1/exp_ds_aniso/channel1/0/0:1024/0:1024/0:16/',
bb,
content_type='application/blosc')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1',
resolution='0',
x_range='0:1024',
y_range='0:1024',
z_range='0:16',
t_range=None)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# Wait for data to be written
request = factory.get(
'/' + version +
'/cutout/col1/exp_ds_aniso/channel1/0/0:1024/0:1024/0:16/',
accepts='application/blosc')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1',
resolution='0',
x_range='0:1024',
y_range='0:1024',
z_range='0:16',
t_range=None).render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
# Trigger downsample
factory = APIRequestFactory()
request = factory.post('/' + version +
'/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# Make Sure status has changed
factory = APIRequestFactory()
request = factory.get('/' + version +
'/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["num_hierarchy_levels"], 5)
self.assertEqual(response.data["status"], "IN_PROGRESS")
for _ in range(0, 30):
# Make request
response = Downsample.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
if response.data["status"] != "IN_PROGRESS":
break
time.sleep(2)
# Verify now downsampled
response = Downsample.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["num_hierarchy_levels"], 5)
self.assertEqual(response.data["status"], "DOWNSAMPLED")
# Get data at res 1 and verify it's non-zero
request = factory.get(
'/' + version +
'/cutout/col1/exp_ds_aniso/channel1/1/0:512/0:512/0:16/',
accepts='application/blosc')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1',
resolution='1',
x_range='0:512',
y_range='0:512',
z_range='0:16',
t_range=None).render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
raw_data = blosc.decompress(response.content)
data_mat_res1_aniso = np.fromstring(raw_data, dtype=np.uint8)
data_mat_res1_aniso = np.reshape(data_mat_res1_aniso, (16, 512, 512),
order='C')
# Make sure not blank
self.assertGreater(data_mat_res1_aniso.sum(), 100)
# Get data at res 1 with iso flag and verify it's non-zero and the same as without flag
request = factory.get(
'/' + version +
'/cutout/col1/exp_ds_aniso/channel1/1/0:512/0:512/0:16/?iso=true',
accepts='application/blosc')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1',
resolution='1',
x_range='0:512',
y_range='0:512',
z_range='0:16',
t_range=None).render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
raw_data = blosc.decompress(response.content)
data_mat_res1_iso = np.fromstring(raw_data, dtype=np.uint8)
data_mat_res1_iso = np.reshape(data_mat_res1_iso, (16, 512, 512),
order='C')
# Make sure not blank
self.assertGreater(data_mat_res1_iso.sum(), 100)
np.testing.assert_array_equal(data_mat_res1_iso, data_mat_res1_aniso)
# Get data at res 4 with iso flag and verify it's non-zero and DIFFERENT than without flag
request = factory.get(
'/' + version +
'/cutout/col1/exp_ds_aniso/channel1/4/0:256/0:256/0:8/?iso=false',
accepts='application/blosc')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1',
resolution='4',
x_range='0:256',
y_range='0:256',
z_range='0:8',
t_range=None).render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
raw_data = blosc.decompress(response.content)
data_mat_res4_aniso = np.fromstring(raw_data, dtype=np.uint8)
data_mat_res4_aniso = np.reshape(data_mat_res4_aniso, (8, 256, 256),
order='C')
# Make sure not blank
self.assertGreater(data_mat_res4_aniso.sum(), 1)
# Get data at res 4 with iso flag and verify it's non-zero and DIFFERENT than without flag
request = factory.get(
'/' + version +
'/cutout/col1/exp_ds_aniso/channel1/4/0:256/0:256/0:8/?iso=true',
accepts='application/blosc')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1',
resolution='4',
x_range='0:256',
y_range='0:256',
z_range='0:8',
t_range=None).render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
raw_data = blosc.decompress(response.content)
data_mat_res4_iso = np.fromstring(raw_data, dtype=np.uint8)
data_mat_res4_iso = np.reshape(data_mat_res4_iso, (8, 256, 256),
order='C')
# Make sure not blank
self.assertGreater(data_mat_res4_iso.sum(), 1)
self.assertRaises(AssertionError, np.testing.assert_array_equal,
data_mat_res4_aniso, data_mat_res4_iso)
# Post data, invalidating the downsample operation
request = factory.post(
'/' + version +
'/cutout/col1/exp_ds_aniso/channel1/0/0:1024/0:1024/0:16/',
bb,
content_type='application/blosc')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1',
resolution='0',
x_range='0:1024',
y_range='0:1024',
z_range='0:16',
t_range=None)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# Verify now NOT downsampled
request = factory.get('/' + version +
'/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
response = Downsample.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["num_hierarchy_levels"], 5)
self.assertEqual(response.data["status"], "NOT_DOWNSAMPLED")
def test_start_and_cancel_downsample_aniso(self, get_msgs_mock, del_msgs_mock, ret_msgs_mock):
chans = self.dbsetup.insert_downsample_data()
job4 = { 'channel_id': 3874 }
job7 = { 'channel_id': chans[0].id }
job9 = { 'channel_id': 3999 }
get_msgs_mock.side_effect = [
[
{ 'MessageId': 'job4', 'ReceiptHandle': 'handle_job4', 'Body': json.dumps(job4) },
{ 'MessageId': 'job7', 'ReceiptHandle': 'handle_job7', 'Body': json.dumps(job7) },
{ 'MessageId': 'job9', 'ReceiptHandle': 'handle_job9', 'Body': json.dumps(job9) },
],
[],
]
exp_return_to_queue_msgs = [get_msgs_mock.side_effect[0][0], get_msgs_mock.side_effect[0][2]]
exp_visibility_change_payload = [
{
f'ChangeMessageVisibilityBatchRequestEntry.{ind+1}.Id': m['MessageId'],
f'ChangeMessageVisibilityBatchRequestEntry.{ind+1}.ReceiptHandle': m['ReceiptHandle'],
f'ChangeMessageVisibilityBatchRequestEntry.{ind+1}.VisibilityTimeout': 0,
} for ind, m in enumerate(exp_return_to_queue_msgs)
]
exp_del_handle = get_msgs_mock.side_effect[0][1]['ReceiptHandle']
factory = APIRequestFactory()
request = factory.post('/' + version + '/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso',
channel='channel1')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# Make Sure status has changed
factory = APIRequestFactory()
request = factory.get('/' + version + '/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso',
channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["num_hierarchy_levels"], 5)
self.assertEqual(response.data["status"], Channel.DownsampleStatus.QUEUED)
# Cancel the downsample job
request = factory.delete('/' + version + '/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso',
channel='channel1')
get_msgs_mock.assert_called_once_with(ANY)
del_msgs_mock.assert_called_once_with(ANY, exp_del_handle)
ret_msgs_mock.assert_called_once_with(ANY, exp_visibility_change_payload)
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
# Make Sure status has changed
factory = APIRequestFactory()
request = factory.get('/' + version + '/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso',
channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], Channel.DownsampleStatus.NOT_DOWNSAMPLED)
# Try to cancel the downsample job again, but it won't because in NOT_DOWNSAMPLED state
request = factory.delete('/' + version + '/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request, collection='col1', experiment='exp_ds_aniso',
channel='channel1')
self.assertEqual(response.status_code, status.HTTP_409_CONFLICT)
def test_start_and_cancel_downsample_aniso(self):
self.dbsetup.insert_downsample_data()
factory = APIRequestFactory()
request = factory.post('/' + version +
'/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# Make Sure status has changed
factory = APIRequestFactory()
request = factory.get('/' + version +
'/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["num_hierarchy_levels"], 5)
self.assertEqual(response.data["status"], "IN_PROGRESS")
# Cancel the downsample job
request = factory.delete('/' + version +
'/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1')
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
# Make Sure status has changed
factory = APIRequestFactory()
request = factory.get('/' + version +
'/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], "NOT_DOWNSAMPLED")
# Try to cancel the downsample job again, but it won't because in NOT_DOWNSAMPLED state
request = factory.delete('/' + version +
'/downsample/col1/exp_ds_aniso/channel1/',
content_type='application/json')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Downsample.as_view()(request,
collection='col1',
experiment='exp_ds_aniso',
channel='channel1')
self.assertEqual(response.status_code, status.HTTP_409_CONFLICT)