def test_dataset_metadata(self):
ds_metadata = self.get_dataset_metadata()
validate_with_schema(ds_metadata, "dataset")
# check missing fields
ds_metadata = self.get_dataset_metadata()
ds_metadata.pop("sources")
with self.assertRaises(ValidationError):
validate_with_schema(ds_metadata, "dataset")
# check invalid fields
ds_metadata = self.get_dataset_metadata()
ds_metadata["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(ds_metadata, "dataset")
ds_metadata = self.get_dataset_metadata()
ds_metadata["sources"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(ds_metadata, "dataset")
ds_metadata = self.get_dataset_metadata()
ds_metadata["views"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(ds_metadata, "dataset")
# check invalid names
ds_metadata = self.get_dataset_metadata()
ds_metadata["sources"]["foo bar"] = metadata.get_image_metadata(
"image2", '/images/image2.xml', file_format="bdv.n5")
with self.assertRaises(ValidationError):
validate_with_schema(ds_metadata, "dataset")
def migrate_source_metadata(name, source, dataset_folder, menu_name):
source_type = source["type"]
xml_locations = source["storage"]
xml_path = os.path.join(dataset_folder, "images", xml_locations["local"])
if source_type in ("image", "mask"):
view = metadata.get_default_view(
"image",
name,
menu_name=menu_name,
color=source["color"],
contrastLimits=source["contrastLimits"])
new_source = metadata.get_image_metadata(dataset_folder, xml_path)
source_type = "image"
else:
assert source_type == "segmentation"
if "tableFolder" in source:
table_location = os.path.join(dataset_folder,
source["tableFolder"])
assert os.path.exists(table_location), table_location
else:
table_location = None
seg_color = source["color"]
seg_color = "glasbey" if seg_color == "randomFromGlasbey" else seg_color
if table_location is None:
view = metadata.get_default_view("segmentation",
name,
menu_name=menu_name,
lut=seg_color)
else:
view = metadata.get_default_view("segmentation",
name,
menu_name=menu_name,
lut=seg_color,
tables=["default.tsv"])
new_source = metadata.get_segmentation_metadata(
dataset_folder, xml_path, table_location=table_location)
if "remote" in xml_locations:
remote_xml = os.path.join("images", xml_locations["remote"])
new_source[source_type]["imageData"]["bdv.n5.s3"] = {
"relativePath": remote_xml
}
has_remote = True
else:
has_remote = False
return new_source, view, has_remote
def get_dataset_metadata(self):
dataset_metadata = {
"is2D": False,
"description": "My dataset.",
"sources": {
"image1":
metadata.get_image_metadata("image1",
"/images/image1.xml",
file_format="bdv.n5"),
"seg1":
metadata.get_segmentation_metadata("seg1",
"/images/seg1.xml",
file_format="bdv.n5")
},
"views": {
"default": metadata.get_default_view("image", "image1")
}
}
return dataset_metadata
def get_source(client, bucket, container, internal_path, endpoint,
dataset_folder, source_name, view, menu_name):
source_object = os.path.join(container, internal_path, 'attributes.json')
attrs_file = s3_utils.download_file(client, bucket, source_object)
with open(attrs_file) as f:
attrs = json.load(f)
name = attrs['name'] if source_name is None else source_name
# for now we hard-code the source type to image.
# in the future it would be nice to infer this somehow from the attributes
source_type = 'image'
# get the mobie source metadata
address = os.path.join(endpoint, bucket, container, internal_path)
if source_type == 'image':
source = metadata.get_image_metadata(dataset_folder,
address,
file_format='openOrganelle.s3')
else:
source = metadata.get_segmentation_metadata(
dataset_folder, address, file_format='openOrganelle.s3')
# get the mobie view metadata
# if the menu-name was not specified, we infer it from the root of the source name
if menu_name is None:
menu_name = os.path.split(internal_path)[0]
menu_name = name if menu_name == "" else menu_name
if view is None:
view = metadata.get_default_view(source_type,
name,
menu_name=menu_name)
else:
view.update({"uiSelectionGroup": menu_name})
validate_view_metadata(view, sources=[name])
return name, source, view
def test_image_source(self):
from mobie.metadata import get_image_metadata
ds_folder = "/path"
xml_path = "/path/to/bdv.xml"
# check valid
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder,
xml_path,
file_format="bdv.n5",
description="My shiny image")
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["imageData"] = {
"bdv.n5": {
"relativePath": "path/to/bdv.xml"
},
"bdv.n5.s3": {
"relativePath": "path/to/some-other.xml"
}
}
validate_with_schema(source, "source")
# check missing fields
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"].pop("imageData")
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["imageData"].pop("bdv.n5")
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["imageData"]["bdv.n5"].pop("relativePath")
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
# check invalid fields
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["imageData"]["foo"] = "bar"
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")
source = get_image_metadata(ds_folder, xml_path, file_format="bdv.n5")
source["image"]["imageData"]["tiff"] = {"path": "my-tiff.tiff"}
with self.assertRaises(ValidationError):
validate_with_schema(source, "source")