def _check_remote_metadata(self, file_format, service_endpoint,
bucket_name):
dataset_folder = os.path.join(self.root, self.dataset_name)
dataset_metadata = read_dataset_metadata(dataset_folder)
validate_with_schema(dataset_metadata, "dataset")
new_file_format = file_format + ".s3"
sources = dataset_metadata["sources"]
for name, source in sources.items():
source_type = list(source.keys())[0]
storage = source[source_type]["imageData"]
self.assertIn(new_file_format, storage)
if new_file_format.startswith("bdv"):
xml = storage[new_file_format]["relativePath"]
xml_path = os.path.join(dataset_folder, xml)
self.assertTrue(os.path.exists(xml_path))
_, ep, bn, _ = parse_s3_xml(xml_path)
self.assertEqual(ep, service_endpoint)
self.assertEqual(bn, bucket_name)
else:
address = storage[new_file_format]["s3Address"]
self.assertTrue(address.startswith(service_endpoint))
proj_metadata = read_project_metadata(self.root)
validate_with_schema(proj_metadata, "project")
file_formats = proj_metadata["imageDataFormats"]
self.assertIn(file_format, file_formats)
self.assertIn(new_file_format, file_formats)
def test_get_view(self):
from mobie.metadata import get_view, get_image_display, get_segmentation_display
# single image views
# with get_image_display arguments
display_args = {"contrastLimits": [0.0, 1.0], "opacity": 1.0}
view = get_view(["my-view"], ["image"], [["my-image"]], [display_args],
is_exclusive=True,
menu_name="bookmark")
validate_with_schema(view, "view")
# with get_image_display return value
display = get_image_display("my-view", ["my-image"], **display_args)
view = get_view(["my-view"], ["image"], [["my-image"]], [display],
is_exclusive=True,
menu_name="bookmark")
validate_with_schema(view, "view")
# single segmentation view
# with get_segmentation_display arguments
display_args = {"lut": "glasbey", "opacity": 1.0}
view = get_view(["my-view"], ["segmentation"], [["my-segmentation"]],
[display_args],
is_exclusive=True,
menu_name="bookmark")
validate_with_schema(view, "view")
# with get_image_display return value
display = get_segmentation_display("my-view", ["my-segmentation"],
**display_args)
view = get_view(["my-view"], ["segmentation"], [["my-segmentation"]],
[display],
is_exclusive=True,
menu_name="bookmark")
validate_with_schema(view, "view")
# multi source view
image_display = get_image_display("images", ["my-image1", "my-image2"],
contrastLimits=[0.0, 2.4],
opacity=1.0)
seg_display = get_segmentation_display("segmentations", ["my-seg"],
opacity=0.6,
lut="glasbey")
view = get_view(["images", "segmentations"], ["image", "segmentation"],
[["my-image1", "my-image2"], ["my-seg"]],
[image_display, seg_display],
is_exclusive=True,
menu_name="bookmark")
validate_with_schema(view, "view")
def test_default_location(self):
ds_metadata = self.get_dataset_metadata()
ds_metadata["defaultLocation"] = {"position": [1.0, 2.0, 3.0]}
validate_with_schema(ds_metadata, "dataset")
ds_metadata = self.get_dataset_metadata()
ds_metadata["defaultLocation"] = {
"position": [1.0, 2.0, 3.0],
"timepoint": 0
}
validate_with_schema(ds_metadata, "dataset")
ds_metadata = self.get_dataset_metadata()
ds_metadata["defaultLocation"] = {
"affine": np.random.rand(12).tolist(),
"timepoint": 42
}
validate_with_schema(ds_metadata, "dataset")
ds_metadata = self.get_dataset_metadata()
ds_metadata["defaultLocation"] = {"gobeldiguk": [99, 83, 4]}
with self.assertRaises(ValidationError):
validate_with_schema(ds_metadata, "dataset")
def test_segmentation_source(self):
from mobie.metadata import get_segmentation_metadata
ds_folder = "/path"
xml_path = "/path/to/bdv.xml"
# check valid
source = get_segmentation_metadata(ds_folder,
xml_path,
file_format="bdv.n5")
validate_with_schema(source, "source")
source = get_segmentation_metadata(ds_folder,
xml_path,
file_format="bdv.n5",
description="My shiny segmentation")
validate_with_schema(source, "source")
source = get_segmentation_metadata(ds_folder,
xml_path,
table_location="/path/to/tables",
file_format="bdv.n5")
validate_with_schema(source, "source")
# check missing fields
source = get_segmentation_metadata(ds_folder,
xml_path,
file_format="bdv.n5")
source["segmentation"].pop("imageData")
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
# check invalid fields
source = get_segmentation_metadata(ds_folder,
xml_path,
file_format="bdv.n5")
source["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_segmentation_metadata(ds_folder,
xml_path,
file_format="bdv.n5")
source["segmentation"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_segmentation_metadata(ds_folder,
xml_path,
file_format="bdv.n5")
source["segmentation"]["imageData"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_segmentation_metadata(ds_folder,
xml_path,
file_format="bdv.n5",
table_location="/path/to/tables")
source["segmentation"]["tableData"]["excel"] = {
"relative_path": "/path/to/tables"
}
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_segmentation_metadata(ds_folder,
xml_path,
file_format="bdv.n5",
table_location="/path/to/tables")
source["segmentation"]["tableData"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
def test_image_source(self):
from mobie.metadata import get_image_metadata
ds_folder = "/path"
xml_path = "/path/to/bdv.xml"
# check valid
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder,
xml_path,
file_format="bdv.n5",
description="My shiny image")
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["imageData"] = {
"bdv.n5": {
"relativePath": "path/to/bdv.xml"
},
"bdv.n5.s3": {
"relativePath": "path/to/some-other.xml"
}
}
validate_with_schema(source, "source")
# check missing fields
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"].pop("imageData")
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["imageData"].pop("bdv.n5")
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["imageData"]["bdv.n5"].pop("relativePath")
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
# check invalid fields
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["imageData"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["imageData"]["tiff"] = {"path": "my-tiff.tiff"}
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
def test_project_metadata(self):
schema = self.get_schema()
metadata = self.get_project_metadata()
validate_with_schema(metadata, schema)
# check optional fields and additional values
metadata = self.get_project_metadata()
metadata["description"] = "Lorem ipsum."
metadata["references"] = ["https://my-publication.com"]
validate_with_schema(metadata, schema)
metadata = self.get_project_metadata()
metadata["imageDataFormats"] = ["bdv.n5", "bdv.n5.s3", "bdv.hdf5"]
validate_with_schema(metadata, schema)
# check missing fields
metadata = self.get_project_metadata()
metadata.pop("datasets")
with self.assertRaises(ValidationError):
validate_with_schema(metadata, schema)
# check invalid fields
metadata = self.get_project_metadata()
metadata["abc"] = "def"
with self.assertRaises(ValidationError):
validate_with_schema(metadata, schema)
# check invalid values
metadata = self.get_project_metadata()
metadata["specVersion"] = "0.3.3"
with self.assertRaises(ValidationError):
validate_with_schema(metadata, "project")
metadata = self.get_project_metadata()
metadata["imageDataFormats"] = ["bdv.n5", "tiff"]
with self.assertRaises(ValidationError):
validate_with_schema(metadata, schema)
def test_transform_grid_view(self):
from mobie.metadata import get_grid_view, get_affine_source_transform
# we need an initial dataset for the grid view
sources = self.init_ds()
grid_sources = [[source] for source in sources]
# only grid transform
view = get_grid_view(self.ds_folder,
"grid-view",
grid_sources,
use_transformed_grid=False,
menu_name="grid")
validate_with_schema(view, "view")
view = get_grid_view(self.ds_folder,
"grid-view",
grid_sources,
use_transformed_grid=True,
menu_name="grid")
validate_with_schema(view, "view")
# additional transforms
trafos = [
get_affine_source_transform([source], np.random.rand(12))
for source in sources
]
view = get_grid_view(self.ds_folder,
"grid-view",
grid_sources,
additional_source_transforms=trafos,
menu_name="grid")
validate_with_schema(view, "view")
# additional transforms, changed source names and grid_sources
trafos = [
get_affine_source_transform(
[source],
np.random.rand(12),
source_names_after_transform=[f"transformed-{ii}"])
for ii, source in enumerate(sources)
]
transformed_sources = [[f"transformed-{ii}"]
for ii in range(len(sources))]
all_transformed_sources = [
trafo for trafos in transformed_sources for trafo in trafos
]
view = get_grid_view(self.ds_folder,
"grid-view",
grid_sources,
additional_source_transforms=trafos,
grid_sources=transformed_sources,
use_transformed_grid=False,
menu_name="grid")
# check that all source displays list the names in transformed sources
for disp in view["sourceDisplays"]:
disp_sources = disp[list(disp.keys())[0]]["sources"]
if isinstance(disp_sources, dict):
disp_sources = list(disp_sources.values())
disp_sources = [
sname for srcs in disp_sources for sname in srcs
]
self.assertTrue(
all(sname in all_transformed_sources
for sname in disp_sources))
validate_with_schema(view, "view")
view = get_grid_view(self.ds_folder,
"grid-view",
grid_sources,
additional_source_transforms=trafos,
grid_sources=transformed_sources,
use_transformed_grid=True,
menu_name="grid")
# check that all source displays list the names in transformed sources
for disp in view["sourceDisplays"]:
disp_sources = disp[list(disp.keys())[0]]["sources"]
if isinstance(disp_sources, dict):
disp_sources = list(disp_sources.values())
disp_sources = [
sname for srcs in disp_sources for sname in srcs
]
self.assertTrue(
all(sname in all_transformed_sources
for sname in disp_sources))
validate_with_schema(view, "view")
def test_source_transforms(self):
from mobie.metadata import (get_affine_source_transform,
get_crop_source_transform,
get_image_display,
get_merged_grid_source_transform,
get_transformed_grid_source_transform,
get_view)
settings = {"contrastLimits": [0.0, 1.0], "opacity": 1.0}
# affine trafo
affine = get_affine_source_transform(
["my-image"],
np.random.rand(12),
timepoints=[0, 1],
source_names_after_transform=["my-transformed-image"])
view = get_view(names=["image-view"],
source_types=["image"],
sources=[["my-transformed-image"]],
display_settings=[settings],
is_exclusive=True,
menu_name="bookmark",
source_transforms=[affine])
validate_with_schema(view, "view")
# crop trafo
crop = get_crop_source_transform(
["my-image"],
np.random.rand(3),
np.random.rand(3),
timepoints=[0, 1],
source_names_after_transform=["my-cropped-image"],
center_at_origin=True,
rectify=True)
view = get_view(names=["image-view"],
source_types=["image"],
sources=[["my-cropped-image"]],
display_settings=[settings],
is_exclusive=True,
menu_name="bookmark",
source_transforms=[crop])
validate_with_schema(view, "view")
# grid trafo
grid = get_merged_grid_source_transform(
["my-image1", "my-image2", "my-image3", "my-image4"],
"merged-images")
view = get_view(
names=["image-grid"],
source_types=["image"],
sources=[["my-image1", "my-image2", "my-image3", "my-image4"]],
display_settings=[settings],
is_exclusive=True,
menu_name="bookmark",
source_transforms=[grid])
validate_with_schema(view, "view")
# transform grid trafo from list
grid = get_transformed_grid_source_transform(
[["my-image1", "my-image2"], ["my-image3", "my-image4"]],
positions=[[0, 0], [1, 1]],
center_at_origin=True)
view = get_view(
names=["image-grid"],
source_types=["image"],
sources=[["my-image1", "my-image2", "my-image3", "my-image4"]],
display_settings=[settings],
is_exclusive=True,
menu_name="bookmark",
source_transforms=[grid])
validate_with_schema(view, "view")
# combined transformations and new display settings
crop = get_crop_source_transform(
["my-transformed-image"],
np.random.rand(3),
np.random.rand(3),
timepoints=[0, 1],
source_names_after_transform=["my-cropped-image"])
settings = get_image_display("image-view", ["my-cropped-image"],
**settings)
view = get_view(names=["image-view"],
source_types=["image"],
sources=[["my-cropped-image"]],
display_settings=[settings],
is_exclusive=True,
menu_name="bookmark",
source_transforms=[affine, crop])
validate_with_schema(view, "view")
def test_viewer_transform(self):
from mobie.metadata import get_default_view
trafos = [
{
"timepoint": 0
},
{
"affine": np.random.rand(12).tolist()
},
{
"affine": np.random.rand(12).tolist(),
"timepoint": 0
},
{
"normalizedAffine": np.random.rand(12).tolist()
},
{
"normalizedAffine": np.random.rand(12).tolist(),
"timepoint": 1
},
{
"position": np.random.rand(3).tolist()
},
{
"position": np.random.rand(3).tolist(),
"timepoint": 2
},
{
"normalVector": np.random.rand(3).tolist()
},
]
for trafo in trafos:
view = get_default_view("image",
"my-image",
viewer_transform=trafo)
validate_with_schema(view, "view")
# test missing fields
invalid_trafos = [{"foo": "bar"}]
for trafo in invalid_trafos:
view = get_default_view("image",
"my-image",
viewer_transform=trafo)
with self.assertRaises(ValidationError):
validate_with_schema(view, "view")
# test invalid fields
invalid_trafos = [{
"timepoint": 0,
"foo": "bar"
}, {
"affine": np.random.rand(12).tolist(),
"foo": "bar"
}, {
"normalizedAffine": np.random.rand(12).tolist(),
"x": "y"
}, {
"position": np.random.rand(3).tolist(),
"a": 3
}]
for trafo in invalid_trafos:
view = get_default_view("image",
"my-image",
viewer_transform=trafo)
with self.assertRaises(ValidationError):
validate_with_schema(view, "view")
# test invalid values
invalid_trafos = [{
"timepoint": -1
}, {
"affine": np.random.rand(11).tolist()
}, {
"normalizedAffine": np.random.rand(13).tolist()
}, {
"position": np.random.rand(4).tolist()
}]
for trafo in invalid_trafos:
view = get_default_view("image",
"my-image",
viewer_transform=trafo)
with self.assertRaises(ValidationError):
validate_with_schema(view, "view")
def test_image_view(self):
from mobie.metadata import get_default_view
# test the default view
view = get_default_view("image", "my-image")
validate_with_schema(view, "view")
# test custom image settings
custom_kwargs = [
{
"contrastLimits": [0., 255.],
"color": "white"
},
{
"contrastLimits": [0., 2000.],
"color": "red"
},
{
"contrastLimits": [-10., 20000000.]
},
{
"showImagesIn3d": True
},
{
"showImagesIn3d": True,
"resolution3dView": [10., 10., 12.]
},
{
"blendingMode": "sumOccluding"
},
]
for kwargs in custom_kwargs:
view = get_default_view("image", "my-image", **kwargs)
validate_with_schema(view, "view")
# test missing fields
view = get_default_view("image", "my-image")
view["sourceDisplays"][0]["imageDisplay"].pop("color")
with self.assertRaises(ValidationError):
validate_with_schema(view, "view")
# test invalid fields
view = get_default_view("image", "my-image")
view["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(view, "view")
view = get_default_view("image", "my-image")
view["sourceDisplays"][0]["imageDisplay"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(view, "view")
# test invalid values
invalid_kwargs = [{
"uiSelectionGroup": "abc a"
}, {
"uiSelectionGroup": "abc/a"
}, {
"uiSelectionGroup": "abc;"
}, {
"color": "foobar"
}, {
"color": "r=1,g=2,b=3,a=4,z=5"
}, {
"contrastLimits": [1., 2., 3.]
}, {
"showImagesIn3d": "foobar"
}, {
"resolution3dView": [1., 2., 3., 4.]
}, {
"blendingMode": "summe"
}]
for kwargs in invalid_kwargs:
if "uiSelectionGroup" in kwargs:
menu_name = kwargs.pop("uiSelectionGroup")
view = get_default_view("image",
"my-image",
menu_name=menu_name,
**kwargs)
with self.assertRaises(ValidationError):
validate_with_schema(view, "view")
def test_segmentation_view(self):
from mobie.metadata import get_default_view
# test the default view
view = get_default_view("segmentation", "my-seg")
validate_with_schema(view, "view")
# test custom segmentation settings
custom_kwargs = [{
"opacity": 0.5,
"lut": "glasbey"
}, {
"opacity": 0.9,
"lut": "viridis",
"colorByColumn": "colname",
"showSelectedSegmentsIn3d": True,
"tables": ["a.csv", "b.tsv"],
"valueLimits": [0, 2500]
}, {
"selectedSegmentIds": ["my-seg;0;1", "my-seg;0;2", "my-seg;1;10"]
}, {
"showAsBoundaries": True,
"boundaryThickness": 12
}]
for kwargs in custom_kwargs:
view = get_default_view("segmentation", "my-seg", **kwargs)
validate_with_schema(view, "view")
# test missing fields
view = get_default_view("segmentation", "my-seg")
view["sourceDisplays"][0]["segmentationDisplay"].pop("opacity")
with self.assertRaises(ValidationError):
validate_with_schema(view, "view")
# test invalid fields
view = get_default_view("segmentation", "my-seg")
view["sourceDisplays"][0]["segmentationDisplay"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(view, "view")
# test invalid values
invalid_kwargs = [{
"opacity": 10
}, {
"lut": "red"
}, {
"lut": "foobar"
}, {
"selectedSegmentIds": ["my-seg,0,2"]
}, {
"selectedSegmentIds": ["my-seg/abc;0;2"]
}, {
"selectedSegmentIds": ["my-segc;abba;2"]
}]
for kwargs in invalid_kwargs:
view = get_default_view("segmentation", "my-seg", **kwargs)
with self.assertRaises(ValidationError):
validate_with_schema(view, "view")
# test view with source transformations
trafos = [{
"parameters": np.random.rand(12).tolist()
}, {
"parameters": np.random.rand(12).tolist(),
"timepoints": [0]
}, {
"parameters": np.random.rand(12).tolist(),
"timepoints": [1, 2, 3]
}]
for trafo in trafos:
view = get_default_view("image",
"my-image",
source_transform=trafo)
validate_with_schema(view, "view")
# test invalid values
invalid_trafos = [{
"parameters": np.random.rand(12).tolist(),
"timepoints": [-1]
}, {
"parameters": np.random.rand(12).tolist(),
"timepoints": [-1, 2, 3]
}]
for trafo in invalid_trafos:
view = get_default_view("image",
"my-image",
source_transform=trafo)
with self.assertRaises(ValidationError):
validate_with_schema(view, "view")
def test_dataset_metadata(self):
ds_metadata = self.get_dataset_metadata()
validate_with_schema(ds_metadata, "dataset")
# check missing fields
ds_metadata = self.get_dataset_metadata()
ds_metadata.pop("sources")
with self.assertRaises(ValidationError):
validate_with_schema(ds_metadata, "dataset")
# check invalid fields
ds_metadata = self.get_dataset_metadata()
ds_metadata["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(ds_metadata, "dataset")
ds_metadata = self.get_dataset_metadata()
ds_metadata["sources"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(ds_metadata, "dataset")
ds_metadata = self.get_dataset_metadata()
ds_metadata["views"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(ds_metadata, "dataset")
# check invalid names
ds_metadata = self.get_dataset_metadata()
ds_metadata["sources"]["foo bar"] = metadata.get_image_metadata(
"image2", '/images/image2.xml', file_format="bdv.n5")
with self.assertRaises(ValidationError):
validate_with_schema(ds_metadata, "dataset")