def test_encoded_stream(self):
data = numpy.random.randint(0, 255, (3, 100, 200)).astype(numpy.uint8)
codec = VoxelsNddataCodec(data.dtype)
stream = codec.create_encoded_stream_from_ndarray(data)
roundtrip_data = codec.decode_to_ndarray(stream, data.shape)
assert roundtrip_data.flags['F_CONTIGUOUS']
self._assert_matching(roundtrip_data, data)
def test_encoded_stream(self):
data = numpy.random.randint(0,255, (3, 100, 200)).astype(numpy.uint8)
codec = VoxelsNddataCodec( data.dtype )
stream = codec.create_encoded_stream_from_ndarray(data)
roundtrip_data = codec.decode_to_ndarray(stream, data.shape)
assert roundtrip_data.flags['F_CONTIGUOUS']
self._assert_matching(roundtrip_data, data)
def test_basic_roundtrip(self):
data = numpy.random.randint(0,255, (3, 100, 200)).astype(numpy.uint8)
codec = VoxelsNddataCodec( data.dtype )
stream = StringIO.StringIO()
codec.encode_from_ndarray(stream, data)
stream.seek(0)
roundtrip_data = codec.decode_to_ndarray(stream, data.shape)
assert roundtrip_data.flags['F_CONTIGUOUS']
self._assert_matching(roundtrip_data, data)
def test_basic_roundtrip(self):
data = numpy.random.randint(0, 255, (3, 100, 200)).astype(numpy.uint8)
codec = VoxelsNddataCodec(data.dtype)
stream = StringIO.StringIO()
codec.encode_from_ndarray(stream, data)
stream.seek(0)
roundtrip_data = codec.decode_to_ndarray(stream, data.shape)
assert roundtrip_data.flags['F_CONTIGUOUS']
self._assert_matching(roundtrip_data, data)
def test_all_dtypes(self):
for dtype in [numpy.uint8, numpy.uint16, numpy.uint32, numpy.float32, numpy.float64]:
data = numpy.random.randint(0,255, (3,100,200)).astype(dtype)
codec = VoxelsNddataCodec( data.dtype )
stream = StringIO.StringIO()
codec.encode_from_ndarray(stream, data)
stream.seek(0)
roundtrip_data = codec.decode_to_ndarray(stream, data.shape)
assert roundtrip_data.flags['F_CONTIGUOUS']
self._assert_matching(roundtrip_data, data)
def test_all_dtypes(self):
for dtype in [
numpy.uint8, numpy.uint16, numpy.uint32, numpy.float32,
numpy.float64
]:
data = numpy.random.randint(0, 255, (3, 100, 200)).astype(dtype)
codec = VoxelsNddataCodec(data.dtype)
stream = StringIO.StringIO()
codec.encode_from_ndarray(stream, data)
stream.seek(0)
roundtrip_data = codec.decode_to_ndarray(stream, data.shape)
assert roundtrip_data.flags['F_CONTIGUOUS']
self._assert_matching(roundtrip_data, data)
def post_ndarray(connection,
uuid,
data_name,
access_type,
voxels_metadata,
start,
stop,
new_data,
throttle=False):
_validate_query_bounds(start,
stop,
voxels_metadata.shape,
allow_overflow_extents=True)
codec = VoxelsNddataCodec(voxels_metadata.dtype)
rest_query = _format_subvolume_rest_uri(uuid,
data_name,
access_type,
start,
stop,
format="",
query_args=None,
throttle=throttle)
body_data_stream = codec.create_encoded_stream_from_ndarray(new_data)
# Slightly tricky here:
# The httplib docs say that we can only send a stream that has a fileno() method,
# but it turns out we can send any stream as long as we provide our own content-length header.
headers = {
"Content-Type": VoxelsNddataCodec.VOLUME_MIMETYPE,
"Content-Length": str(codec.calculate_buffer_len(new_data.shape))
}
connection.request("POST",
rest_query,
body=body_data_stream,
headers=headers)
with contextlib.closing(connection.getresponse()) as response:
#if response.status != httplib.NO_CONTENT:
if response.status != httplib.OK:
raise DvidHttpError("subvolume post",
response.status, response.reason,
response.read(), "POST", rest_query,
"<binary data>", headers)
# Something (either dvid or the httplib) gets upset if we don't read the full response.
response.read()
def get_ndarray( connection, uuid, data_name, access_type, voxels_metadata, start, stop, query_args=None, throttle=False ):
_validate_query_bounds( start, stop, voxels_metadata.shape )
codec = VoxelsNddataCodec( voxels_metadata.dtype )
response = get_subvolume_response( connection, uuid, data_name, access_type, start, stop, query_args=query_args, throttle=throttle )
with contextlib.closing(response):
# "Full" roi shape includes channel axis and ALL channels
full_roi_shape = numpy.array(stop) - start
full_roi_shape[0] = voxels_metadata.shape[0]
decoded_data = codec.decode_to_ndarray( response, full_roi_shape )
# Was the response fully consumed? Check.
# NOTE: This last read() is not optional.
# Something in the http implementation gets upset if we read out the exact amount we needed.
# That is, we MUST read beyond the end of the stream. So, here we go.
excess_data = response.read()
if excess_data:
# Uh-oh, we expected it to be empty.
raise UnexpectedResponseError( "Received data was longer than expected by {} bytes. (Expected only {} bytes.)"
"".format( len(excess_data), len(numpy.getbuffer(decoded_data)) ) )
# Select the requested channels from the returned data.
return decoded_data
def get_ndarray( connection, uuid, data_name, access_type, voxels_metadata, start, stop, query_args=None, throttle=False ):
_validate_query_bounds( start, stop, voxels_metadata.shape )
codec = VoxelsNddataCodec( voxels_metadata.dtype )
response = get_subvolume_response( connection, uuid, data_name, access_type, start, stop, query_args=query_args, throttle=throttle )
with contextlib.closing(response):
# "Full" roi shape includes channel axis and ALL channels
full_roi_shape = numpy.array(stop) - start
full_roi_shape[0] = voxels_metadata.shape[0]
decoded_data = codec.decode_to_ndarray( response, full_roi_shape )
# Was the response fully consumed? Check.
# NOTE: This last read() is not optional.
# Something in the http implementation gets upset if we read out the exact amount we needed.
# That is, we MUST read beyond the end of the stream. So, here we go.
excess_data = response.read()
if excess_data:
# Uh-oh, we expected it to be empty.
raise UnexpectedResponseError( "Received data was longer than expected by {} bytes. (Expected only {} bytes.)"
"".format( len(excess_data), len(numpy.getbuffer(decoded_data)) ) )
# Select the requested channels from the returned data.
return decoded_data
def post_ndarray( connection, uuid, data_name, access_type, voxels_metadata, start, stop, new_data, throttle=False ):
_validate_query_bounds( start, stop, voxels_metadata.shape, allow_overflow_extents=True )
codec = VoxelsNddataCodec( voxels_metadata.dtype )
rest_query = _format_subvolume_rest_uri( uuid, data_name, access_type, start, stop, format="", query_args=None, throttle=throttle )
body_data_stream = codec.create_encoded_stream_from_ndarray(new_data)
# Slightly tricky here:
# The httplib docs say that we can only send a stream that has a fileno() method,
# but it turns out we can send any stream as long as we provide our own content-length header.
headers = { "Content-Type" : VoxelsNddataCodec.VOLUME_MIMETYPE,
"Content-Length" : str(codec.calculate_buffer_len(new_data.shape)) }
connection.request( "POST", rest_query, body=body_data_stream, headers=headers )
with contextlib.closing( connection.getresponse() ) as response:
#if response.status != httplib.NO_CONTENT:
if response.status != httplib.OK:
raise DvidHttpError(
"subvolume post", response.status, response.reason, response.read(),
"POST", rest_query, "<binary data>", headers)
# Something (either dvid or the httplib) gets upset if we don't read the full response.
response.read()
