def test_get_cubes(self):
"""Test adding cubes to the cache"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Add items
morton_id = [112, 125, 516]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)
rkv.put_cubes(keys, data)
# Get cube
cubes = rkv.get_cubes(keys)
assert len(cubes) == 3
for m, c, d in zip(morton_id, cubes, data):
assert c[0] == m
assert c[1] == 0
data_retrieved = blosc.unpack_array(c[2])
np.testing.assert_array_equal(data_retrieved, blosc.unpack_array(d))
def test_put_cubes(self):
"""Test adding cubes to the cache"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Make sure there are no cuboids in the cache
keys = self.cache_client.keys('CACHED-CUBOID&{}&{}*'.format(self.resource.get_lookup_key(), resolution))
assert not keys
# Add items
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], [123, 124, 126])
rkv.put_cubes(keys, data)
db_keys = self.cache_client.keys('CACHED-CUBOID*')
db_keys = [x.decode() for x in db_keys]
assert len(set(keys)) == 3
for k, d in zip(keys, db_keys):
assert k in db_keys
def test_get_cubes(self):
"""Test adding cubes to the cache"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Add items
morton_id = [112, 125, 516]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
morton_id)
rkv.put_cubes(keys, data)
# Get cube
cubes = rkv.get_cubes(keys)
assert len(cubes) == 3
for m, c, d in zip(morton_id, cubes, data):
assert c[0] == m
assert c[1] == 0
data_retrieved = blosc.unpack_array(c[2])
np.testing.assert_array_equal(data_retrieved,
blosc.unpack_array(d))
def supercube_compatibility(self, super_cube):
super_cube = blosc.unpack_array(super_cube)
if len(super_cube.shape) == 3:
return blosc.pack_array(super_cube.reshape((1,) + super_cube.shape))
else:
return blosc.pack_array(super_cube)
def updateIndex ( self, ch, entityid, index, resolution ):
"""Updated the database index table with the input index hash table"""
curindex = self.getIndex(ch, entityid, resolution, True)
if curindex == []:
if self.NPZ:
fileobj = cStringIO.StringIO ()
np.save ( fileobj, index )
self.kvio.putIndex(ch, entityid, resolution, fileobj.getvalue())
else:
self.kvio.putIndex(ch, entityid, resolution, blosc.pack_array(index))
else :
# Update Index to the union of the currentIndex and the updated index
newIndex = np.union1d(curindex, index)
# Update the index in the database
if self.NPZ:
fileobj = cStringIO.StringIO ()
np.save ( fileobj, newIndex )
self.kvio.putIndex(ch, entityid, resolution, fileobj.getvalue(), True)
else:
self.kvio.putIndex(ch, entityid, resolution, blosc.pack_array(newIndex), True)
def __read_img_label_into_zipnp(self,img_label_path_dic,img_label_dic):
pbar = pb.ProgressBar(widgets=[pb.Percentage(), pb.Bar(), pb.ETA()], maxval=len(img_label_path_dic)).start()
count = 0
for fn, img_label_path in img_label_path_dic.items():
img_label_np_dic = {}
img_sitk, original_spacing, original_sz = self.__read_and_clean_itk_info(img_label_path['image'])
resized_img, resize_factor = self.resize_img(img_sitk)
img_np = sitk.GetArrayFromImage(resized_img)
img_np = self.normalize_intensity(img_np)
img_label_np_dic['image'] = blosc.pack_array(img_np.astype(np.float32))
if self.has_label:
label_sitk, _, _ = self.__read_and_clean_itk_info(img_label_path['label'])
resized_label,_ = self.resize_img(label_sitk,is_label=True)
label_np = sitk.GetArrayFromImage(resized_label)
label_index = list(np.unique(label_np))
img_label_np_dic['label'] = blosc.pack_array(label_np.astype(np.int64))
img_label_np_dic['label_index'] = label_index
img_after_resize = self.img_after_resize if self.img_after_resize is not None else original_sz
new_spacing= original_spacing*(original_sz-1)/(np.array(img_after_resize)-1)
normalized_spacing = self._normalize_spacing(new_spacing,img_after_resize, silent_mode=True)
img_label_np_dic['original_sz'] =original_sz
img_label_np_dic['original_spacing'] = original_spacing
img_label_np_dic['spacing'] = normalized_spacing
img_label_np_dic['img_sz'] = list(img_np.shape)
img_label_dic[fn] =img_label_np_dic
count +=1
pbar.update(count)
pbar.finish()
def updateIndex(self, ch, entityid, index, resolution):
"""Updated the database index table with the input index hash table"""
curindex = self.getIndex(ch, entityid, resolution, True)
if curindex == []:
if self.NPZ:
fileobj = cStringIO.StringIO()
np.save(fileobj, index)
self.kvio.putIndex(ch, entityid, resolution,
fileobj.getvalue())
else:
self.kvio.putIndex(ch, entityid, resolution,
blosc.pack_array(index))
else:
# Update Index to the union of the currentIndex and the updated index
newIndex = np.union1d(curindex, index)
# Update the index in the database
if self.NPZ:
fileobj = cStringIO.StringIO()
np.save(fileobj, newIndex)
self.kvio.putIndex(ch, entityid, resolution,
fileobj.getvalue(), True)
else:
self.kvio.putIndex(ch, entityid, resolution,
blosc.pack_array(newIndex), True)
def test_put_cubes(self):
"""Test adding cubes to the cache"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Make sure there are no cuboids in the cache
keys = self.cache_client.keys('CACHED-CUBOID&{}&{}*'.format(
self.resource.get_lookup_key(), resolution))
assert not keys
# Add items
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
[123, 124, 126])
rkv.put_cubes(keys, data)
db_keys = self.cache_client.keys('CACHED-CUBOID*')
db_keys = [x.decode() for x in db_keys]
assert len(set(keys)) == 3
for k, d in zip(keys, db_keys):
assert k in db_keys
def supercube_compatibility(self, super_cube):
super_cube = blosc.unpack_array(super_cube)
if len(super_cube.shape) == 3:
return blosc.pack_array(
super_cube.reshape((1, ) + super_cube.shape))
else:
return blosc.pack_array(super_cube)
def render(self, data, media_type=None, renderer_context=None):
if not data["data"].data.flags['C_CONTIGUOUS']:
data["data"].data = np.ascontiguousarray(data["data"].data, dtype=data["data"].data.dtype)
# Return data, squeezing time dimension if only a single point
if data["time_request"]:
return blosc.pack_array(data["data"].data)
else:
return blosc.pack_array(np.squeeze(data["data"].data, axis=(0,)))
def send_data(socket, *inputs):
"""Pack the array and send it"""
if len(inputs) == 1:
packed_array = blosc.pack_array(inputs[0])
return socket.send((packed_array, ))
elif len(inputs) == 4:
packed_array = blosc.pack_array(inputs[0])
return socket.send((packed_array, ) + (inputs[1:]))
def render(self, data, media_type=None, renderer_context=None):
if not data.data.flags['C_CONTIGUOUS']:
data.data = data.data.copy(order='C')
# Return data, squeezing time dimension if only a single point
try:
return blosc.pack_array(np.squeeze(data.data, axis=(0, )))
except ValueError:
# The are more than 1 time points so don't squeeze.
return blosc.pack_array(data.data)
def stepEnv(self, action=None):
# If no manual action was specified by the user
if action is None:
action = random.randint(0, self.env.action_space.n - 1)
action = int(action)
obs, reward, done, info = self.env.step(action)
self.current_actions.append(action)
self.current_images.append(self.lastObs['image'])
self.current_directions.append(self.lastObs['direction'])
self.showEnv(obs)
self.lastObs = obs
if done:
if reward > 0: # i.e. we did not lose
if self.shift < len(self.demos):
self.demos[self.shift] = self.current_demo, self.shift
else:
self.demos.append(
(self.current_mission,
blosc.pack_array(np.array(self.current_images)),
self.current_directions, self.current_actions))
utils.save_demos(self.demos, self.demos_path)
self.missionBox.append('Demonstrations are saved.')
utils.synthesize_demos(self.demos)
self.shift += 1
self.resetEnv()
else:
self.shiftEnv()
def breakCubes(self, timestamp, super_zidx, resolution, super_cube):
"""Breaking the supercuboids into cuboids"""
super_cube = blosc.unpack_array(super_cube)
# print "breaking supercube shape: {}".format(super_cube.shape)
# Empty lists for zindx and cube data
zidx_list = []
cube_list = []
# SuperCube Size
[xnumcubes, ynumcubes, znumcubes] = self.db.datasetcfg.supercube_size
# Cube dimensions
cubedim = self.db.datasetcfg.get_cubedim(resolution)
[x,y,z] = MortonXYZ(super_zidx)
# start = map(mul, cubedim, [x,y,z])
start = map(mul, [x,y,z], self.db.datasetcfg.supercube_size)
for z in range(znumcubes):
for y in range(ynumcubes):
for x in range(xnumcubes):
zidx = XYZMorton(map(add, start, [x,y,z]))
# Parameters in the cube slab
index = map(mul, cubedim, [x,y,z])
end = map(add, index, cubedim)
cube_data = super_cube[:,index[2]:end[2], index[1]:end[1], index[0]:end[0]]
zidx_list.append(zidx)
# print "mini cube:", cube_data.shape
cube_list.append(blosc.pack_array(cube_data))
return zidx_list, [timestamp]*len(zidx_list), cube_list
def postBlosc(p, post_data, time=False, neariso=False, direct=False):
"""Post data using blosc packed numpy array"""
# Build the url and then create a npz object
if time:
url = 'https://{}/sd/{}/{}/blosc/{}/{},{}/{},{}/{},{}/{},{}/'.format(
SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args)
elif p.channels is not None:
url = 'https://{}/sd/{}/{}/blosc/{}/{},{}/{},{}/{},{}/'.format(
SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args)
elif p.channels is None:
url = 'https://{}/sd/{}/blosc/{}/{},{}/{},{}/{},{}/'.format(
SITE_HOST, p.token, p.resolution, *p.args)
if neariso:
url = url + NEARISO
if direct:
url = url + DIRECT
try:
# Build a post request
resp = postURL(url, blosc.pack_array(post_data))
return resp
except Exception as e:
return e
def singleThreadTest(self, start_value, size_iterations):
"""Generate the URL for read test"""
min_values = [xmin,ymin,zmin] = map(add, self.info_interface.offset(self.resolution), start_value)
max_values = map(add, min_values, self.info.cuboid_dimension(self.resolution))
range_args = [None]*(len(min_values)+len(max_values))
for i in range(0, size_iterations, 1):
if all([a<b for a,b in zip(max_values, self.info_interface.image_size(self.resolution))]):
range_args[::2] = min_values
range_args[1::2] = max_values
# print "Size", reduce(mul, map(sub, max_values, min_values), 1)*2.0/(1024*1024)
if self.write_tests:
data = blosc.pack_array(np.ones( [len(self.channels)]+map(sub, range_args[1::2], range_args[::2])[::-1], dtype=self.datatype) * random.randint(0,255))
self.data_list.append(data)
self.fetch_list.append(generateURLBlosc(self.host_name, self.token_name, self.channels, self.resolution, range_args, direct=self.direct))
else:
self.fetch_list.append(generateURLBlosc(self.host_name, self.token_name, self.channels, self.resolution, range_args, direct=self.direct))
# min_values = max_values
# max_values = map(add, map(sub, min_values, temp_values), min_values)
min_values[2] = max_values[2]
max_values[2] = min_values[2] + self.info.cuboid_dimension(self.resolution)[2]
max_values[(i+1)%2] = start_value[(i+1)%2] + (max_values[(i+1)%2]-min_values[(i+1)%2])*2
else:
break
def _post_cutout_no_chunking_blosc(self, token, channel,
x_start, y_start, z_start,
data, resolution):
"""
Accepts data in zyx. !!!
"""
data = numpy.expand_dims(data, axis=0)
blosc_data = blosc.pack_array(data)
url = self.url("{}/{}/blosc/{}/{},{}/{},{}/{},{}/".format(
token, channel,
resolution,
x_start, x_start + data.shape[3],
y_start, y_start + data.shape[2],
z_start, z_start + data.shape[1]
))
req = requests.post(url, data=blosc_data, headers={
'Content-Type': 'application/octet-stream'
})
if req.status_code is not 200:
raise RemoteDataUploadError(req.text)
else:
return True
def test_delete_cube(self):
"""Test cube delete method"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data = [data_packed1]
# Add items
morton_id = [112]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)
result = rkv.cube_exists(keys[0])
assert not result
rkv.put_cubes(keys, data)
result = rkv.cube_exists(keys[0])
assert result
rkv.delete_cube(keys[0])
result = rkv.cube_exists(keys[0])
assert not result
def test_is_dirty(self):
"""Test cube delete method"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data = [data_packed1]
# Add item to cached-cuboid
morton_id = [112]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)
rkv.put_cubes(keys, data)
result = rkv.is_dirty(keys)
assert not result[0]
# Fake a write
write_cuboid_key = "WRITE-CUBOID&{}".format(keys[0].split('&', 1)[1])
write_cuboid_key, time_sample, morton = write_cuboid_key.rsplit('&', 2)
rkv.insert_cube_in_write_buffer(write_cuboid_key, time_sample, morton, data[0])
result = rkv.is_dirty(keys)
assert result[0]
def test_delete_cube(self):
"""Test cube delete method"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data = [data_packed1]
# Add items
morton_id = [112]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
morton_id)
result = rkv.cube_exists(keys[0])
assert not result
rkv.put_cubes(keys, data)
result = rkv.cube_exists(keys[0])
assert result
rkv.delete_cube(keys[0])
result = rkv.cube_exists(keys[0])
assert not result
def cutout_create(
self, resource, resolution, x_range, y_range, z_range, time_range, numpyVolume,
url_prefix, auth, session, send_opts):
"""Upload a cutout to the Boss data store.
Args:
resource (ndio.ndresource.resource.Resource): Resource compatible with cutout operations.
resolution (int): 0 indicates native resolution.
x_range (string): x range such as '10:20' which means x>=10 and x<20.
y_range (string): y range such as '10:20' which means y>=10 and y<20.
z_range (string): z range such as '10:20' which means z>=10 and z<20.
time_range (string): None or time range such as 30:40 which means t>=30 and t<40.
numpyVolume (numpy.array): A 3D or 4D (time) numpy matrix in (time)ZYX order.
url_prefix (string): Protocol + host such as https://api.theboss.io
auth (string): Token to send in the request header.
session (requests.Session): HTTP session to use for request.
send_opts (dictionary): Additional arguments to pass to session.send().
"""
compressed = blosc.pack_array(numpyVolume)
req = self.get_cutout_request(
resource, 'POST', 'application/blosc-python',
url_prefix, auth,
resolution, x_range, y_range, z_range, time_range, compressed)
prep = session.prepare_request(req)
resp = session.send(prep, **send_opts)
if resp.status_code == 201:
return
msg = ('Create cutout failed on {}, got HTTP response: ({}) - {}'.format(
resource.name, resp.status_code, resp.text))
raise HTTPError(msg, request = req, response = resp)
def test_is_dirty(self):
"""Test cube delete method"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data = [data_packed1]
# Add item to cached-cuboid
morton_id = [112]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
morton_id)
rkv.put_cubes(keys, data)
result = rkv.is_dirty(keys)
assert not result[0]
# Fake a write
write_cuboid_key = "WRITE-CUBOID&{}".format(keys[0].split('&', 1)[1])
write_cuboid_key, time_sample, morton = write_cuboid_key.rsplit('&', 2)
rkv.insert_cube_in_write_buffer(write_cuboid_key, time_sample, morton,
data[0])
result = rkv.is_dirty(keys)
assert result[0]
def __get_clean_label(self,img_label_dict, img_name_list):
"""
:param img_label_dict:
:param img_name_list:
:return:
"""
print(" Attention, the annotation for background is assume to be 0 ! ")
print(" Attention, we are using the union set of the label! ")
if self.interested_label_list is None:
interested_label_set = set()
for i, fname in enumerate(img_name_list):
label_set = img_label_dict[fname]['label_index']
if i ==0:
interested_label_set = set(label_set)
else:
interested_label_set = interested_label_set.union(label_set)
interested_label_list = list(interested_label_set)
else:
interested_label_list = self.interested_label_list
#self.standard_label_index = tuple([int(item) for item in interested_label_list])
for fname in img_name_list:
label = img_label_dict[fname]['label']
label = self.__convert_to_standard_label_map(label, interested_label_list)
label_density = list(np.bincount(label.reshape(-1).astype(np.int32)) / len(label.reshape(-1)))
img_label_dict[fname]['label'] = blosc.pack_array(label)
img_label_dict[fname]['label_density']=label_density
img_label_dict[fname]['label_org_index'] = interested_label_list
img_label_dict[fname]['label_converted_index'] = list(range(len(interested_label_list)))
return img_label_dict
def postBlosc(p, post_data):
"""Post data using the blosc interface"""
# Build the url and then create a hdf5 object
url = 'http://{}/{}/{}/blosc/{}/{},{}/{},{}/{},{}/'.format(
SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args)
return (url, blosc.pack_array(post_data))
def test_channel_uint16_wrong_dimensions_numpy(self):
""" Test posting with the wrong xyz dims using the numpy interface"""
test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128))
test_mat = test_mat.astype(np.uint16)
bb = blosc.pack_array(test_mat)
# Create request
factory = APIRequestFactory()
request = factory.post(
'/' + version + '/cutout/col1/exp1/channel2/0/0:100/0:128/0:16/',
bb,
content_type='application/blosc-python')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp1',
channel='channel2',
resolution='0',
x_range='0:100',
y_range='0:128',
z_range='0:16',
t_range=None)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_cutout_get_success(self, mock_session):
resolution = 0
x_range = '20:40'
y_range = '50:70'
z_range = '30:50'
time_range = '10:25'
url_prefix = 'https://api.theboss.io'
auth = 'mytoken'
fake_prepped_req = PreparedRequest()
fake_prepped_req.headers = {}
mock_session.prepare_request.return_value = fake_prepped_req
data = numpy.random.randint(0, 3000, (15, 20, 20, 20), numpy.uint16)
compressed_data = blosc.pack_array(data)
fake_response = Response()
fake_response.status_code = 200
fake_response._content = compressed_data
mock_session.send.return_value = fake_response
send_opts = {}
actual = self.vol.cutout_get(
self.chan, resolution, x_range, y_range, z_range, time_range,
url_prefix, auth, mock_session, send_opts)
numpy.testing.assert_array_equal(data, actual)
def uploadExistingProject(self, channel_name, resolution, start_values, neariso=False):
"""Upload an existing project to S3"""
self.setupNewProject()
db = SpatialDB(self.proj)
# checking for channels
if channel_name is None:
channel_list = None
else:
channel_list = [channel_name]
# iterating over channels in a project
for ch in self.proj.projectChannels(channel_list):
# creating the channel resource
self.resource_interface.createChannel(ch.channel_name)
# ingest 1 or more resolutions based on user input
if resolution is None:
start_res = self.proj.datasetcfg.scalinglevels
stop_res = ch.resolution - 1
else:
start_res = resolution
stop_res = resolution - 1
# iterating over resolution
for cur_res in range(start_res, stop_res, -1):
# get the source database sizes
[image_size, time_range] = self.proj.datasetcfg.dataset_dim(cur_res)
[xcubedim, ycubedim, zcubedim] = cubedim = self.proj.datasetcfg.get_cubedim(cur_res)
offset = self.proj.datasetcfg.get_offset(cur_res)
[xsupercubedim, ysupercubedim, zsupercubedim] = supercubedim = self.proj.datasetcfg.get_supercubedim(cur_res)
# set the limits for iteration on the number of cubes in each dimension
xlimit = (image_size[0]-1) / (xsupercubedim) + 1
ylimit = (image_size[1]-1) / (ysupercubedim) + 1
zlimit = (image_size[2]-1) / (zsupercubedim) + 1
# [xlimit, ylimit, zlimit] = limit = self.proj.datasetcfg.get_supercube_limit(cur_res)
[x_start, y_start, z_start] = map(div, start_values, supercubedim)
for z in range(z_start, zlimit, 1):
for y in range(y_start, ylimit, 1):
for x in range(x_start, xlimit, 1):
try:
# cutout the data at the current resolution
data = db.cutout(ch, [x*xsupercubedim, y*ysupercubedim, z*zsupercubedim], [xsupercubedim, ysupercubedim, zsupercubedim], cur_res).data
# generate the morton index
morton_index = XYZMorton([x, y, z])
self.logger.info("[{},{},{}] at res {}".format(x*xsupercubedim, y*ysupercubedim, z*zsupercubedim, cur_res))
# updating the index
# self.cuboidindex_db.putItem(ch.channel_name, cur_res, x, y, z, ch.time_range[0])
# inserting the cube
self.s3_io.putCube(ch, ch.time_stamp[0], morton_index, cur_res, blosc.pack_array(data), neariso=neariso)
except Exception as e:
# checkpoint the ingest
self.logger.error(e)
self.checkpoint_ingest(ch.channel_name, cur_res, x, y, z, e)
raise e
def upload(self, file_name, channel_name, resolution, x_index, y_index, z_index, dimensions=[1, 64, 512,512], time_index=0, neariso=False):
"""Upload a 4D supercuboid directly to dynamo and s3"""
cuboid_data = np.fromfile(file_name, dtype=self.info_interface.get_channel_datatype(channel_name))
cuboid_data = cuboid_data.reshape(dimensions)
super_zidx = XYZMorton([x_index, y_index, z_index])
self.logger.info("Inserting cube {},{},{}".format(x_index, y_index, z_index))
self.cuboidindex_db.putItem(channel_name, resolution, x_index, y_index, z_index, time_index, neariso=neariso)
self.cuboid_bucket.putObject(channel_name, resolution, super_zidx, time_index, blosc.pack_array(cuboid_data), neariso=neariso)
def toBlosc ( self ):
"""Pack the object"""
try:
# Create the compressed cube
return blosc.pack_array(self.data)
except:
logger.error ("Failed to compress database cube. Data integrity concern.")
raise
def putIndex ( self, ch, entityid, timestamp, resolution, index, update=False ):
"""Write the index for the annotation with id"""
if self.NPZ:
fileobj = cStringIO.StringIO ()
np.save ( fileobj, index )
self.kvio.putIndex(ch, entityid, timestamp, resolution, fileobj.getvalue(), update )
else:
self.kvio.putIndex(ch, entityid, timestamp, resolution, blosc.pack_array(index), update)
def test_channel_uint64_cuboid_unaligned_offset_time_offset_blosc_numpy(
self):
""" Test uint64 data, not cuboid aligned, offset, time samples, blosc interface
Test Requires >=2GB of memory!
"""
test_mat = np.random.randint(1, 2**50, (3, 17, 300, 500))
test_mat = test_mat.astype(np.uint64)
bb = blosc.pack_array(test_mat)
# Create request
factory = APIRequestFactory()
request = factory.post(
'/' + version +
'/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203',
bb,
content_type='application/blosc-python')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp1',
dataset='layer1',
resolution='0',
x_range='100:600',
y_range='450:750',
z_range='20:37')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# Create Request to get data you posted
request = factory.get(
'/' + version +
'/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203',
HTTP_ACCEPT='application/blosc-python')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp1',
dataset='layer1',
resolution='0',
x_range='100:600',
y_range='450:750',
z_range='20:37').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
# Decompress
data_mat = blosc.unpack_array(response.content)
# Test for data equality (what you put in is what you got back!)
np.testing.assert_array_equal(data_mat, test_mat)
def toBlosc(self):
"""Pack the object"""
try:
# Create the compressed cube
return blosc.pack_array(self.data)
except:
logger.error(
"Failed to compress database cube. Data integrity concern.")
raise
def putIndex ( self, ch, entityid, resolution, index, update=False ):
"""Write the index for the annotation with id"""
if self.NPZ:
fileobj = cStringIO.StringIO ()
np.save ( fileobj, index )
self.kvio.putIndex(ch, entityid, resolution, fileobj.getvalue(), update)
else:
self.kvio.putIndex(ch, entityid, resolution, blosc.pack_array(index), update)
def generate_dagger_demos(env_name, seeds, fail_obss, fail_actions, mean_steps):
env = gym.make(env_name)
agent = BotAgent(env)
demos = []
for i in range(len(fail_obss)):
# Run the expert for one episode
env.seed(int(seeds[i]))
new_obs = env.reset()
agent.on_reset()
env0_str = env.__str__()
actions = []
images = []
directions = []
debug_info = {'seed': [int(seeds[i])], 'actions': []}
try:
for j in range(min(int(args.dagger_trim_coef * mean_steps), len(fail_obss[i]) - 1)):
obs = fail_obss[i][j]
assert check_obss_equality(obs, new_obs), "Observations {} of seed {} don't match".format(j, seeds[i])
mission = obs['mission']
action = agent.act(update_internal_state=False)['action']
_ = agent.bot.take_action(fail_actions[i][j])
debug_info['actions'].append(fail_actions[i][j])
new_obs, reward, done, _ = env.step(fail_actions[i][j])
if done and reward > 0:
raise ValueError(
"The baby's actions shouldn't solve the task. Env0 {}, Env9{}, Seed {}, actions {}.".format(
env0_str, env.__str__(), int(seeds[i]), fail_actions[i]
))
actions.append(action)
images.append(obs['image'])
directions.append(obs['direction'])
if args.continue_dagger:
obs = new_obs
while not done:
action = agent.act(obs)['action']
debug_info['actions'].append(action)
new_obs, reward, done, _ = env.step(action)
agent.analyze_feedback(reward, done)
actions.append(action)
images.append(obs['image'])
directions.append(obs['direction'])
print(debug_info, actions)
demos.append((mission, blosc.pack_array(np.array(images)), directions, actions))
except Exception as e:
logger.exception("error while generating demo #{}: {}. Env0 {}, Env9{}, Seed {}, actions {}.".format(
len(demos), e, env0_str, env.__str__(), int(seeds[i]), fail_actions[i]))
continue
return demos
def test_channel_uint16_cuboid_aligned_offset_no_time_blosc_numpy(self):
""" Test uint16 data, cuboid aligned, offset, no time samples, blosc interface"""
test_mat = np.random.randint(1, 2**16 - 1, (16, 128, 128))
test_mat = test_mat.astype(np.uint16)
bb = blosc.pack_array(test_mat)
# Create request
factory = APIRequestFactory()
request = factory.post(
'/' + version +
'/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/',
bb,
content_type='application/blosc-python')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp1',
channel='channel2',
resolution='0',
x_range='128:256',
y_range='256:384',
z_range='16:32',
t_range=None)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# Create Request to get data you posted
request = factory.get(
'/' + version +
'/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/',
HTTP_ACCEPT='application/blosc-python')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request,
collection='col1',
experiment='exp1',
channel='channel2',
resolution='0',
x_range='128:256',
y_range='256:384',
z_range='16:32',
t_range=None).render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
# Decompress
data_mat = blosc.unpack_array(response.content)
# Test for data equality (what you put in is what you got back!)
np.testing.assert_array_equal(data_mat, test_mat)
def blosc_up():
for file in file_list:
img_name=photo_test+'\\'+file
img=Image.open(img_name)
# numpy화
# numpy.asarry() 이미지->array
image_array=np.asarray(img)
# pack_array 배열 압축
compressed_bytes = blosc.pack_array(image_array)
xrayimg={"imagenum":file, "photo":compressed_bytes}
client.kaggle1.blosc.insert_one(xrayimg)
def mergeCubes(data1, data2): """Merge Cubes""" import ndlib data1 = blosc.unpack_array(data1) data2 = blosc.unpack_array(data2) # Call vectorize function # vec_func = np.vectorize(lambda x,y: x if y == 0 else y) # Call ctype function ndlib.overwriteMerge_ctype(data1, data2) return blosc.pack_array(data1)
def test_put_object(self):
"""Testing put object"""
cube_data = blosc.pack_array(np.zeros(settings.SUPER_CUBOID_SIZE))
for morton_index in range(0, 10, 1):
self.cuboid_bucket.putObject(nd_proj.channel_name,
nd_proj.resolution, morton_index,
cube_data)
for morton_index in range(0, 10, 1):
supercuboid_key = self.cuboid_bucket.generateSupercuboidKey(
nd_proj.channel_name, nd_proj.resolution, morton_index)
self.cuboid_bucket.deleteObject(supercuboid_key)
def test_put_object_by_key(self):
hashm = hashlib.md5()
hashm.update(b"test_cuboidbucket_data")
cube_data = blosc.pack_array(np.zeros(settings.SUPER_CUBOID_SIZE))
for morton_index in range(0, 10, 1):
key = "{}&{}".format(hashm.hexdigest(), morton_index)
self.cuboid_bucket.putObjectByKey(key, cube_data)
for morton_index in range(0, 10, 1):
key = "{}&{}".format(hashm.hexdigest(), morton_index)
self.cuboid_bucket.deleteObject(key)
def PostBlosc (p, post_data):
"""Post data using the npz interface"""
# Build the url and then create a npz object
url = 'http://{}/blaze/{}/{}/blosc/{}/{},{}/{},{}/{},{}/'.format(SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args)
try:
# Build a post request
req = urllib2.Request(url, blosc.pack_array(post_data))
response = urllib2.urlopen(req)
return response
except urllib2.HTTPError,e:
return e
def test_param_constructor(self):
"""Re-run a testing using the parameter based constructor"""
config = {
"cache_host": self.config["aws"]["cache"],
"cache_db": 1,
"read_timeout": 86400
}
rkv = RedisKVIO(config)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Add items
morton_id = [112, 125, 516]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
morton_id)
rkv.put_cubes(keys, data)
# Get cube
cubes = rkv.get_cubes(keys)
cube = [x for x in cubes]
assert len(cube) == 3
for m, c, d in zip(morton_id, cube, data):
assert c[0] == m
assert c[1] == 0
data_retrieved = blosc.unpack_array(c[2])
np.testing.assert_array_equal(data_retrieved,
blosc.unpack_array(d))
def parse(self, response):
for article in response.css("article.post"):
for image in article.css("img.attachment-post-thumbnail"):
with urllib.request.urlopen(
image.xpath('@src').get()) as img_res:
image_array = np.asarray(bytearray(img_res.read()),
dtype=np.uint8)
compressed_img = blosc.pack_array(image_array)
yield {
'id': article.attrib['id'],
'src': image.xpath('@src').get(),
'img': compressed_img
}
def test_param_constructor(self):
"""Re-run a testing using the parameter based constructor"""
config = {
"cache_host": self.config["aws"]["cache"],
"cache_db": 1,
"read_timeout": 86400
}
rkv = RedisKVIO(config)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Add items
morton_id = [112, 125, 516]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)
rkv.put_cubes(keys, data)
# Get cube
cubes = rkv.get_cubes(keys)
cube = [x for x in cubes]
assert len(cube) == 3
for m, c, d in zip(morton_id, cube, data):
assert c[0] == m
assert c[1] == 0
data_retrieved = blosc.unpack_array(c[2])
np.testing.assert_array_equal(data_retrieved, blosc.unpack_array(d))
def breakCubes(key, blosc_data):
"""break the cubes into smaller chunks"""
key_array = [token, channel_name, res, x1, x2, y1, y2, z1, z2, time_stamp] = key.split('_')
[res, x1, x2, y1, y2, z1, z2] = [int(i) for i in key_array[2:][:-1]]
if blosc_data is None:
return
voxarray = blosc.unpack_array(blosc_data)
br = BlazeRedis(token, channel_name, res)
ds = Dataset(token)
ch = ds.getChannelObj(channel_name)
[zimagesz, yimagesz, ximagesz] = ds.imagesz[res]
#[xcubedim, ycubedim, zcubedim] = cubedim = ds.cubedim[res]
[xcubedim, ycubedim, zcubedim] = cubedim = CUBE_DIM
[xoffset, yoffset, zoffset] = ds.offset[res]
# Calculating the corner and dimension
corner = [x1, y1, z1]
dim = voxarray.shape[::-1][:-1]
# Round to the nearest largest cube in all dimensions
[xstart, ystart, zstart] = start = map(div, corner, cubedim)
znumcubes = (corner[2]+dim[2]+zcubedim-1)/zcubedim - zstart
ynumcubes = (corner[1]+dim[1]+ycubedim-1)/ycubedim - ystart
xnumcubes = (corner[0]+dim[0]+xcubedim-1)/xcubedim - xstart
numcubes = [xnumcubes, ynumcubes, znumcubes]
offset = map(mod, corner, cubedim)
data_buffer = np.zeros(map(mul, numcubes, cubedim)[::-1], dtype=voxarray.dtype)
end = map(add, offset, dim)
data_buffer[offset[2]:end[2], offset[1]:end[1], offset[0]:end[0]] = voxarray
cube_list = []
for z in range(znumcubes):
for y in range(ynumcubes):
for x in range(xnumcubes):
zidx = XYZMorton(map(add, start, [x,y,z]))
# Parameters in the cube slab
index = map(mul, cubedim, [x,y,z])
end = map(add, index, cubedim)
cube_data = data_buffer[index[2]:end[2], index[1]:end[1], index[0]:end[0]]
cube_list.append((br.generateSIKey(zidx), blosc.pack_array(cube_data.reshape((1,)+cube_data.shape))))
return cube_list[:]
def test_write_buffer_io(self):
"""Test methods specific to single cube write buffer io"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data_packed = blosc.pack_array(data1)
key = rkv.insert_cube_in_write_buffer("WRITE-CUBOID&4&1&1&1", 3, 234, data_packed)
data_rx = rkv.get_cube_from_write_buffer(key)
assert data_packed == data_rx
def from_array(array):
"""
Export a numpy array to a blosc array.
Arguments:
array: The numpy array to compress to blosc array
Returns:
Bytes/String. A blosc compressed array
"""
try:
raw_data = blosc.pack_array(array)
except Exception as e:
raise ValueError("Could not compress data from array. {}".format(e))
return raw_data
def generate_demos(env_name, seeds):
env = gym.make(env_name)
raise(NotImplementedError, 'RL expert goes here')
#TODO: insert neural agent here (RL expert) (Mathijs)
#agent = BotAgent(env)
demos = []
for seed in seeds:
# Run the expert for one episode
done = False
env.seed(int(seed))
obs = env.reset()
agent.on_reset()
actions = []
mission = obs["mission"]
images = []
directions = []
try:
while not done:
action = agent.act(obs)['action']
new_obs, reward, done, _ = env.step(action)
agent.analyze_feedback(reward, done)
actions.append(action)
images.append(obs['image'])
directions.append(obs['direction'])
obs = new_obs
if reward > 0:
demos.append((mission, blosc.pack_array(np.array(images)), directions, actions))
if reward == 0:
logger.info("failed to accomplish the mission")
except Exception:
logger.exception("error while generating demo #{}".format(len(demos)))
continue
# logger.info("demo #{}".format(len(demos)))
return demos
def multiThreadTest(self, start_value, size_iterations, number_of_processes):
"""Generate the URL for multi-thread test"""
# min_values = [xmin,ymin,zmin] = map(add, self.info_interface.offset(self.resolution), start_value)
min_values = [xmin, ymin, zmin] = self.info_interface.offset(self.resolution)
# max_values = map(add, min_values, self.info_interface.cuboid_dimension(self.resolution))
max_values = map(add, min_values, map(lambda x: x*self.scale, self.info_interface.supercuboid_dimension(self.resolution)))
range_args = [None]*(len(min_values)+len(max_values))
size_args = [None]*(len(min_values)+len(max_values))
# determine the size of the cutout
for i in range(0, size_iterations, 1):
if all([a<b for a,b in zip(max_values, self.info_interface.image_size(self.resolution))]):
size_args[::2] = min_values
size_args[1::2] = max_values
# increase in x,y
max_values[(i+1)%2] = (max_values[(i+1)%2]-min_values[(i+1)%2])*2
else:
break
# intialize the range args
range_args[:] = size_args[:]
range_args[::2] = map(add, start_value, size_args[::2])
range_args[1::2] = map(add, start_value, size_args[1::2])
# creating a fetch list for each process
for i in range(0, number_of_processes, 1):
# checking if the x,y,z dimensions are exceeded
if all([a<b for a,b in zip(range_args[1::2], self.info_interface.image_size(self.resolution))]):
if self.write_tests:
data = blosc.pack_array(np.ones( [len(self.channels)]+map(sub, range_args[1::2], range_args[::2])[::-1], dtype=self.datatype) * random.randint(0,255))
self.data_list.append(data)
self.fetch_list.append(generateURLBlosc(self.host_name, self.token_name, self.channels, self.resolution, range_args, direct=self.direct))
else:
self.fetch_list.append(generateURLBlosc(self.host_name, self.token_name, self.channels, self.resolution, range_args, direct=self.direct))
# checking if this exceeds the x,y image size
if all([a<b for a,b in zip(map(add, range_args[1:-1:2], size_args[1:-1:2]), self.info_interface.image_size(self.resolution))]):
range_args[0:-2:2] = range_args[1:-1:2]
range_args[1:-1:2] = map(add, range_args[1:-1:2], size_args[1:-1:2])
# if you cannot expand more in x,y then expand in z
elif range_args[-1] < self.info_interface.image_size(self.resolution)[-1]:
range_args[-2] = range_args[-1]
range_args[-1] = range_args[-1] + size_args[-1]
else:
break
else:
break
def test_channel_uint16_wrong_data_type_numpy(self):
""" Test posting the wrong bitdepth data using the blosc-numpy interface"""
test_mat = np.random.randint(1, 2 ** 16 - 1, (16, 128, 128))
test_mat = test_mat.astype(np.uint8)
bb = blosc.pack_array(test_mat)
# Create request
factory = APIRequestFactory()
request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/0:128/0:128/0:16/', bb,
content_type='application/blosc-python')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2',
resolution='0', x_range='0:128', y_range='0:128', z_range='0:16', t_range=None)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def postBlaze (p, post_data, time=False):
"""Post data using npz"""
# Build the url and then create a npz object
if time:
url = 'https://{}/sd/{}/{}/blaze/{}/{},{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
elif p.channels is not None:
url = 'https://{}/sd/{}/{}/blaze/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
elif p.channels is None:
url = 'https://{}/sd/{}/blaze/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, p.resolution, *p.args )
try:
# Build a post request
resp = postURL(url, blosc.pack_array(post_data))
return resp
except Exception as e:
return e
def postBlosc (p, post_data, time=False):
"""Post data using npz"""
# Build the url and then create a npz object
if time:
url = 'http://{}/ca/{}/{}/blosc/{}/{},{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
elif p.channels is not None:
url = 'http://{}/ca/{}/{}/blosc/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
elif p.channels is None:
url = 'http://{}/ca/{}/blosc/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, p.resolution, *p.args )
try:
# Build a post request
req = urllib2.Request(url,blosc.pack_array(post_data))
response = urllib2.urlopen(req)
return response
except urllib2.HTTPError,e:
return e
def test_channel_uint64_time_npygz_download(self):
""" Test uint8 data, using the npygz interface with time series support
"""
test_mat = np.random.randint(1, 256, (3, 4, 128, 128))
test_mat = test_mat.astype(np.uint64)
bb = blosc.pack_array(test_mat)
# Create request
factory = APIRequestFactory()
request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13', bb,
content_type='application/blosc-python')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1',
resolution='0', x_range='0:128', y_range='0:128', z_range='0:4',
t_range='10:13')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# Create Request to get data you posted
request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/0:128/0:128/0:4/10:13',
HTTP_ACCEPT='application/npygz')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1',
resolution='0', x_range='0:128', y_range='0:128', z_range='0:4',
t_range='10:13').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
# Decompress
data_bytes = zlib.decompress(response.content)
# Open
data_obj = io.BytesIO(data_bytes)
data_mat = np.load(data_obj)
# Test for data equality (what you put in is what you got back!)
np.testing.assert_array_equal(data_mat, test_mat)
def insert_tile(self, zoom, row, col, data):
with self.db_connection as db_connection:
cursor = db_connection.cursor()
compression = self.compression
data_type = self.data_type
if compression and (data_type == "xray"):
db_connection.text_factory = str
data_compressed = blosc.pack_array(data, cname=compression)
data = data_compressed
if data_type == "image/TIFF":
try:
assert data.dtype == "uint8"
except:
raise TypeError("dtype %s not supported" % data.dtype)
image = Image.fromarray(np.uint8(data))
buf = ioBuffer()
if compression == "tiff_lzw":
TiffImagePlugin.WRITE_LIBTIFF = True
image.save(buf, "TIFF", compression=compression)
TiffImagePlugin.WRITE_LIBTIFF = False
else:
image.save(buf, "TIFF", compression=compression)
buf.seek(0)
data = Binary(buf.read())
if data_type == "image/JPEG2000":
try:
assert data.dtype == "uint8"
except:
raise TypeError("dtype %s not supported" % data.dtype)
image = Image.fromarray(np.uint8(data))
buf = ioBuffer()
image.save(buf, "j2k")
buf.seek(0)
data = Binary(buf.read())
try:
cursor.execute("""
INSERT INTO tiles
(zoom_level, tile_row, tile_column, tile_data)
VALUES (?,?,?,?)
""", (zoom, row, col, data))
except:
raise
def test_channel_uint64_cuboid_unaligned_offset_time_offset_blosc_numpy(self):
""" Test uint64 data, not cuboid aligned, offset, time samples, blosc interface
Test Requires >=2GB of memory!
"""
test_mat = np.random.randint(1, 256, (3, 17, 300, 500))
test_mat = test_mat.astype(np.uint64)
bb = blosc.pack_array(test_mat)
# Create request
factory = APIRequestFactory()
request = factory.post('/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203', bb,
content_type='application/blosc-python')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1',
resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='200:203')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# Create Request to get data you posted
request = factory.get('/' + version + '/cutout/col1/exp1/layer1/0/100:600/450:750/20:37/200:203',
HTTP_ACCEPT='application/blosc-python')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='layer1',
resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range='200:203').render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
# Decompress
data_mat = blosc.unpack_array(response.content)
# Test for data equality (what you put in is what you got back!)
np.testing.assert_array_equal(data_mat, test_mat)
def postBlosc (p, post_data, time=False, neariso=False, direct=False):
"""Post data using blosc packed numpy array"""
# Build the url and then create a npz object
if time:
url = 'https://{}/sd/{}/{}/blosc/{}/{},{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
elif p.channels is not None:
url = 'https://{}/sd/{}/{}/blosc/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, ','.join(p.channels), p.resolution, *p.args )
elif p.channels is None:
url = 'https://{}/sd/{}/blosc/{}/{},{}/{},{}/{},{}/'.format ( SITE_HOST, p.token, p.resolution, *p.args )
if neariso:
url = url + NEARISO
if direct:
url = url + DIRECT
try:
# Build a post request
resp = postURL(url,blosc.pack_array(post_data))
return resp
except Exception as e:
return e
def test_channel_uint16_cuboid_aligned_offset_no_time_blosc_numpy(self):
""" Test uint16 data, cuboid aligned, offset, no time samples, blosc interface"""
test_mat = np.random.randint(1, 2**16-1, (16, 128, 128))
test_mat = test_mat.astype(np.uint16)
bb = blosc.pack_array(test_mat)
# Create request
factory = APIRequestFactory()
request = factory.post('/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/', bb,
content_type='application/blosc-python')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2',
resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# Create Request to get data you posted
request = factory.get('/' + version + '/cutout/col1/exp1/channel2/0/128:256/256:384/16:32/',
HTTP_ACCEPT='application/blosc-python')
# log in user
force_authenticate(request, user=self.user)
# Make request
response = Cutout.as_view()(request, collection='col1', experiment='exp1', channel='channel2',
resolution='0', x_range='128:256', y_range='256:384', z_range='16:32', t_range=None).render()
self.assertEqual(response.status_code, status.HTTP_200_OK)
# Decompress
data_mat = blosc.unpack_array(response.content)
# Test for data equality (what you put in is what you got back!)
np.testing.assert_array_equal(data_mat, test_mat)