def test_encoded_stream(self): data = numpy.random.randint(0, 255, (3, 100, 200)).astype(numpy.uint8) codec = VoxelsNddataCodec(data.dtype) stream = codec.create_encoded_stream_from_ndarray(data) roundtrip_data = codec.decode_to_ndarray(stream, data.shape) assert roundtrip_data.flags['F_CONTIGUOUS'] self._assert_matching(roundtrip_data, data)
def test_encoded_stream(self): data = numpy.random.randint(0,255, (3, 100, 200)).astype(numpy.uint8) codec = VoxelsNddataCodec( data.dtype ) stream = codec.create_encoded_stream_from_ndarray(data) roundtrip_data = codec.decode_to_ndarray(stream, data.shape) assert roundtrip_data.flags['F_CONTIGUOUS'] self._assert_matching(roundtrip_data, data)
def test_basic_roundtrip(self): data = numpy.random.randint(0,255, (3, 100, 200)).astype(numpy.uint8) codec = VoxelsNddataCodec( data.dtype ) stream = StringIO.StringIO() codec.encode_from_ndarray(stream, data) stream.seek(0) roundtrip_data = codec.decode_to_ndarray(stream, data.shape) assert roundtrip_data.flags['F_CONTIGUOUS'] self._assert_matching(roundtrip_data, data)
def test_basic_roundtrip(self): data = numpy.random.randint(0, 255, (3, 100, 200)).astype(numpy.uint8) codec = VoxelsNddataCodec(data.dtype) stream = StringIO.StringIO() codec.encode_from_ndarray(stream, data) stream.seek(0) roundtrip_data = codec.decode_to_ndarray(stream, data.shape) assert roundtrip_data.flags['F_CONTIGUOUS'] self._assert_matching(roundtrip_data, data)
def test_all_dtypes(self): for dtype in [numpy.uint8, numpy.uint16, numpy.uint32, numpy.float32, numpy.float64]: data = numpy.random.randint(0,255, (3,100,200)).astype(dtype) codec = VoxelsNddataCodec( data.dtype ) stream = StringIO.StringIO() codec.encode_from_ndarray(stream, data) stream.seek(0) roundtrip_data = codec.decode_to_ndarray(stream, data.shape) assert roundtrip_data.flags['F_CONTIGUOUS'] self._assert_matching(roundtrip_data, data)
def test_all_dtypes(self): for dtype in [ numpy.uint8, numpy.uint16, numpy.uint32, numpy.float32, numpy.float64 ]: data = numpy.random.randint(0, 255, (3, 100, 200)).astype(dtype) codec = VoxelsNddataCodec(data.dtype) stream = StringIO.StringIO() codec.encode_from_ndarray(stream, data) stream.seek(0) roundtrip_data = codec.decode_to_ndarray(stream, data.shape) assert roundtrip_data.flags['F_CONTIGUOUS'] self._assert_matching(roundtrip_data, data)
def post_ndarray(connection, uuid, data_name, access_type, voxels_metadata, start, stop, new_data, throttle=False): _validate_query_bounds(start, stop, voxels_metadata.shape, allow_overflow_extents=True) codec = VoxelsNddataCodec(voxels_metadata.dtype) rest_query = _format_subvolume_rest_uri(uuid, data_name, access_type, start, stop, format="", query_args=None, throttle=throttle) body_data_stream = codec.create_encoded_stream_from_ndarray(new_data) # Slightly tricky here: # The httplib docs say that we can only send a stream that has a fileno() method, # but it turns out we can send any stream as long as we provide our own content-length header. headers = { "Content-Type": VoxelsNddataCodec.VOLUME_MIMETYPE, "Content-Length": str(codec.calculate_buffer_len(new_data.shape)) } connection.request("POST", rest_query, body=body_data_stream, headers=headers) with contextlib.closing(connection.getresponse()) as response: #if response.status != httplib.NO_CONTENT: if response.status != httplib.OK: raise DvidHttpError("subvolume post", response.status, response.reason, response.read(), "POST", rest_query, "<binary data>", headers) # Something (either dvid or the httplib) gets upset if we don't read the full response. response.read()
def get_ndarray( connection, uuid, data_name, access_type, voxels_metadata, start, stop, query_args=None, throttle=False ): _validate_query_bounds( start, stop, voxels_metadata.shape ) codec = VoxelsNddataCodec( voxels_metadata.dtype ) response = get_subvolume_response( connection, uuid, data_name, access_type, start, stop, query_args=query_args, throttle=throttle ) with contextlib.closing(response): # "Full" roi shape includes channel axis and ALL channels full_roi_shape = numpy.array(stop) - start full_roi_shape[0] = voxels_metadata.shape[0] decoded_data = codec.decode_to_ndarray( response, full_roi_shape ) # Was the response fully consumed? Check. # NOTE: This last read() is not optional. # Something in the http implementation gets upset if we read out the exact amount we needed. # That is, we MUST read beyond the end of the stream. So, here we go. excess_data = response.read() if excess_data: # Uh-oh, we expected it to be empty. raise UnexpectedResponseError( "Received data was longer than expected by {} bytes. (Expected only {} bytes.)" "".format( len(excess_data), len(numpy.getbuffer(decoded_data)) ) ) # Select the requested channels from the returned data. return decoded_data
def post_ndarray( connection, uuid, data_name, access_type, voxels_metadata, start, stop, new_data, throttle=False ): _validate_query_bounds( start, stop, voxels_metadata.shape, allow_overflow_extents=True ) codec = VoxelsNddataCodec( voxels_metadata.dtype ) rest_query = _format_subvolume_rest_uri( uuid, data_name, access_type, start, stop, format="", query_args=None, throttle=throttle ) body_data_stream = codec.create_encoded_stream_from_ndarray(new_data) # Slightly tricky here: # The httplib docs say that we can only send a stream that has a fileno() method, # but it turns out we can send any stream as long as we provide our own content-length header. headers = { "Content-Type" : VoxelsNddataCodec.VOLUME_MIMETYPE, "Content-Length" : str(codec.calculate_buffer_len(new_data.shape)) } connection.request( "POST", rest_query, body=body_data_stream, headers=headers ) with contextlib.closing( connection.getresponse() ) as response: #if response.status != httplib.NO_CONTENT: if response.status != httplib.OK: raise DvidHttpError( "subvolume post", response.status, response.reason, response.read(), "POST", rest_query, "<binary data>", headers) # Something (either dvid or the httplib) gets upset if we don't read the full response. response.read()