def test_save_numpy_array(tmpdir): tmpdir = str(tmpdir) h, w = 30, 30 c = np.random.randint(low=1, high=4) z = np.random.randint(low=1, high=6) # 2D images without channel axis img = _get_image(h, w, 1) img = np.squeeze(img) files = utils.save_numpy_array(img, 'name', '/a/b/', tmpdir) assert len(files) == 1 for f in files: assert os.path.isfile(f) assert f.startswith(os.path.join(tmpdir, 'a', 'b')) # 2D images img = _get_image(h, w, c) files = utils.save_numpy_array(img, 'name', '/a/b/', tmpdir) assert len(files) == c for f in files: assert os.path.isfile(f) assert f.startswith(os.path.join(tmpdir, 'a', 'b')) # 3D images imgs = np.vstack([_get_image(h, w, c)[None, ...] for i in range(z)]) files = utils.save_numpy_array(imgs, 'name', '/a/b/', tmpdir) assert len(files) == c for f in files: assert os.path.isfile(f) assert f.startswith(os.path.join(tmpdir, 'a', 'b')) # Bad path will not fail, but will log error img = _get_image(h, w, c) files = utils.save_numpy_array(img, 'name', '/a/b/', '/does/not/exist/') assert not files
def test_grpc_response_to_dict(): # pylint: disable=E1101 # test valid response data = _get_image(300, 300, 1) tensor_proto = grpc_clients.make_tensor_proto(data, 'DT_FLOAT') response = PredictResponse() response.outputs['prediction'].CopyFrom(tensor_proto) response_dict = grpc_clients.grpc_response_to_dict(response) assert isinstance(response_dict, (dict, )) np.testing.assert_allclose(response_dict['prediction'], data) # test scalar input data = 3 tensor_proto = grpc_clients.make_tensor_proto(data, 'DT_FLOAT') response = PredictResponse() response.outputs['prediction'].CopyFrom(tensor_proto) response_dict = grpc_clients.grpc_response_to_dict(response) assert isinstance(response_dict, (dict, )) np.testing.assert_allclose(response_dict['prediction'], data) # test bad dtype # logs an error, but should throw a KeyError as well. data = _get_image(300, 300, 1) tensor_proto = grpc_clients.make_tensor_proto(data, 'DT_FLOAT') response = PredictResponse() response.outputs['prediction'].CopyFrom(tensor_proto) response.outputs['prediction'].dtype = 32 with pytest.raises(KeyError): response_dict = grpc_clients.grpc_response_to_dict(response)
def test_get_image_label(self, mocker, redis_client): queue = 'q' stg = DummyStorage() consumer = consumers.SegmentationConsumer(redis_client, stg, queue) image = _get_image(256, 256, 1) # test no label provided expected = 1 mocker.patch.object(consumer, 'detect_label', lambda *x: expected) label = consumer.get_image_label(None, image, 'some hash') assert label == expected # test label provided expected = 2 label = consumer.get_image_label(expected, image, 'some hash') assert label == expected # test label provided is invalid with pytest.raises(ValueError): label = -1 consumer.get_image_label(label, image, 'some hash') # test label provided is bad type with pytest.raises(ValueError): label = 'badval' consumer.get_image_label(label, image, 'some hash')
def _write_image(filepath, img_w=300, img_h=300): imarray = _get_image(img_h, img_w, 1) _, ext = os.path.splitext(filepath.lower()) if ext in {'.tif', '.tiff'}: tifffile.imsave(filepath, imarray[..., 0]) else: img = array_to_img(imarray, scale=False, data_format='channels_last') img.save(filepath)
def test_make_tensor_proto(): # test with numpy array data = _get_image(300, 300, 1) proto = grpc_clients.make_tensor_proto(data, 'DT_FLOAT') assert isinstance(proto, (TensorProto, )) # test with value data = 10.0 proto = grpc_clients.make_tensor_proto(data, types_pb2.DT_FLOAT) assert isinstance(proto, (TensorProto, ))
def test__consume(self, mocker, redis_client): queue = 'track' storage = DummyStorage() test_hash = 0 dummy_results = { 'y_tracked': np.zeros((32, 32, 1)), 'tracks': [] } mock_model = Bunch( get_batch_size=lambda *x: 1, input_shape=(1, 32, 32, 1) ) mock_app = Bunch( predict=lambda *x, **y: dummy_results, track=lambda *x, **y: dummy_results, model_mpp=1, model=mock_model, ) consumer = consumers.CalibanConsumer(redis_client, storage, queue) mocker.patch.object(consumer, 'get_grpc_app', lambda *x, **y: mock_app) # mock get_model_wrapper for neighborhood encoder mocker.patch.object(consumer, 'get_model_wrapper', lambda *x, **y: mock_model) frames = 3 dummy_data = { 'X': np.array([_get_image(21, 21) for _ in range(frames)]), 'y': np.random.randint(0, 9, size=(frames, 21, 21)), } mocker.patch.object(consumer, '_load_data', lambda *x: dummy_data) # test finished statuses are returned for status in (consumer.failed_status, consumer.final_status): test_hash += 1 data = {'input_file_name': 'file.tiff', 'status': status} redis_client.hmset(test_hash, data) result = consumer._consume(test_hash) assert result == status # test new key is processed test_hash += 1 data = {'input_file_name': 'file.tiff', 'status': 'new'} redis_client.hmset(test_hash, data) result = consumer._consume(test_hash) assert result == consumer.final_status assert redis_client.hget(test_hash, 'status') == consumer.final_status
def test_detect_label(self, mocker, redis_client): # pylint: disable=W0613 shape = (1, 256, 256, 1) queue = 'q' consumer = consumers.SegmentationConsumer(redis_client, None, queue) expected_label = random.randint(1, 9) mock_app = Bunch(predict=lambda *x, **y: expected_label, model=Bunch(get_batch_size=lambda *x: 1)) mocker.patch.object(consumer, 'get_grpc_app', lambda *x: mock_app) image = _get_image(shape[1] * 2, shape[2] * 2, shape[3]) mocker.patch.object(settings, 'LABEL_DETECT_ENABLED', False) label = consumer.detect_label(image) assert label == 0 mocker.patch.object(settings, 'LABEL_DETECT_ENABLED', True) label = consumer.detect_label(image) assert label == expected_label
def test_detect_scale(self, mocker, redis_client): # pylint: disable=W0613 shape = (1, 256, 256, 1) consumer = consumers.MesmerConsumer(redis_client, None, 'q') image = _get_image(shape[1] * 2, shape[2] * 2, shape[3]) expected_scale = 1 # random.uniform(0.5, 1.5) # model_mpp = random.uniform(0.5, 1.5) mock_app = Bunch( predict=lambda *x, **y: expected_scale, # model_mpp=model_mpp, model=Bunch(get_batch_size=lambda *x: 1)) mocker.patch.object(consumer, 'get_grpc_app', lambda *x: mock_app) mocker.patch.object(settings, 'SCALE_DETECT_ENABLED', False) scale = consumer.detect_scale(image) assert scale == 1 # model_mpp mocker.patch.object(settings, 'SCALE_DETECT_ENABLED', True) scale = consumer.detect_scale(image) assert scale == expected_scale # * model_mpp
def test__load_data(self, tmpdir, mocker, redis_client): queue = 'track' storage = DummyStorage() consumer = consumers.CalibanConsumer(redis_client, storage, queue) tmpdir = str(tmpdir) exp = random.randint(0, 99) # test load trk files key = 'trk file test' mocker.patch('redis_consumer.utils.load_track_file', lambda x: exp) result = consumer._load_data(key, tmpdir, 'data.trk') assert result == exp result = consumer._load_data(key, tmpdir, 'data.trks') assert result == exp # test bad filetype key = 'invalid filetype test' with pytest.raises(ValueError): consumer._load_data(key, tmpdir, 'data.npz') # test bad ndim for tiffstack fname = 'test.tiff' filepath = os.path.join(tmpdir, fname) tifffile.imsave(filepath, _get_image()) with pytest.raises(ValueError): consumer._load_data(key, tmpdir, fname) # test successful workflow def hget_successful_status(*_): return consumer.final_status def hget_failed_status(*_): return consumer.failed_status def write_child_tiff(*_, **__): letters = string.ascii_lowercase name = ''.join(random.choice(letters) for i in range(12)) path = os.path.join(tmpdir, '{}.tiff'.format(name)) tifffile.imsave(path, _get_image(21, 21)) return [path] mocker.patch.object(settings, 'INTERVAL', 0) mocker.patch.object(redis_client, 'hget', hget_successful_status) mocker.patch('redis_consumer.utils.iter_image_archive', write_child_tiff) for label_detect in (True, False): mocker.patch.object(settings, 'SCALE_DETECT_ENABLED', label_detect) mocker.patch.object(settings, 'LABEL_DETECT_ENABLED', label_detect) tifffile.imsave(filepath, np.random.random((3, 21, 21))) results = consumer._load_data(key, tmpdir, fname) X, y = results.get('X'), results.get('y') assert isinstance(X, np.ndarray) assert isinstance(y, np.ndarray) assert X.shape == y.shape # test failed child with pytest.raises(RuntimeError): mocker.patch.object(redis_client, 'hget', hget_failed_status) consumer._load_data(key, tmpdir, fname) # test wrong number of images in the test file with pytest.raises(RuntimeError): mocker.patch.object(redis_client, 'hget', hget_successful_status) mocker.patch('redis_consumer.utils.iter_image_archive', lambda *x: range(1, 3)) consumer._load_data(key, tmpdir, fname)
def write_child_tiff(*_, **__): letters = string.ascii_lowercase name = ''.join(random.choice(letters) for i in range(12)) path = os.path.join(tmpdir, '{}.tiff'.format(name)) tifffile.imsave(path, _get_image(21, 21)) return [path]