def nwb2asset(
nwb_path: Union[str, Path],
digest: Optional[Digest] = None,
schema_version: Optional[str] = None,
) -> models.BareAsset:
if schema_version is not None:
current_version = models.get_schema_version()
if schema_version != current_version:
raise ValueError(
f"Unsupported schema version: {schema_version}; expected {current_version}"
)
start_time = datetime.now().astimezone()
metadata = get_metadata(nwb_path)
if digest is not None:
metadata["digest"] = digest.value
metadata["digest_type"] = digest.algorithm.name
metadata["contentSize"] = op.getsize(nwb_path)
metadata["encodingFormat"] = "application/x-nwb"
metadata["dateModified"] = get_utcnow_datetime()
metadata["blobDateModified"] = ensure_datetime(os.stat(nwb_path).st_mtime)
metadata["path"] = str(nwb_path)
if metadata["blobDateModified"] > metadata["dateModified"]:
lgr.warning("mtime %s of %s is in the future",
metadata["blobDateModified"], nwb_path)
asset = metadata2asset(metadata)
asset = process_ndtypes(asset, metadata["nd_types"])
end_time = datetime.now().astimezone()
if asset.wasGeneratedBy is None:
asset.wasGeneratedBy = []
asset.wasGeneratedBy.append(get_generator(start_time, end_time))
return asset
def test_check_schema_version_matches_default() -> None:
responses.add(
responses.GET,
"https://test.nil/api/info/",
json={"schema_version": get_schema_version()},
)
client = DandiAPIClient("https://test.nil/api")
client.check_schema_version()
def __init__(self, path, allow_empty=False, schema_version=None):
if schema_version is not None:
current_version = get_schema_version()
if schema_version != current_version:
raise ValueError(
f"Unsupported schema version: {schema_version}; expected {current_version}"
)
super().__init__(path, allow_empty=allow_empty)
def validate_dandi_nwb(filepath, schema_version=None, devel_debug=False):
"""Provide validation of .nwb file regarding requirements we impose"""
if schema_version is not None:
from dandischema.models import BareAsset, get_schema_version
from pydantic import ValidationError
from .metadata import nwb2asset
current_version = get_schema_version()
if schema_version != current_version:
raise ValueError(
f"Unsupported schema version: {schema_version}; expected {current_version}"
)
try:
asset = nwb2asset(filepath,
digest=32 * "d" + "-1",
digest_type="dandi_etag")
BareAsset(**asset.dict())
except ValidationError as e:
if devel_debug:
raise
lgr.warning("Validation error for %s: %s",
filepath,
e,
extra={"validating": True})
return [str(e)]
except Exception as e:
if devel_debug:
raise
lgr.warning(
"Unexpected validation error for %s: %s",
filepath,
e,
extra={"validating": True},
)
return [f"Failed to read metadata: {e}"]
return []
else:
# make sure that we have some basic metadata fields we require
try:
meta = get_metadata(filepath)
except Exception as e:
if devel_debug:
raise
lgr.warning(
"Failed to read metadata in %s: %s",
filepath,
e,
extra={"validating": True},
)
return [f"Failed to read metadata: {e}"]
return _check_required_fields(meta, _required_nwb_metadata_fields)
def __init__(
self,
path: Union[str, Path],
allow_empty: bool = False,
schema_version: Optional[str] = None,
) -> None:
if schema_version is not None:
current_version = get_schema_version()
if schema_version != current_version:
raise ValueError(
f"Unsupported schema version: {schema_version}; expected {current_version}"
)
super().__init__(path, allow_empty=allow_empty)
def nwb2asset(
nwb_path: Union[str, Path],
digest: Optional[Digest] = None,
schema_version: Optional[str] = None,
) -> models.BareAsset:
if schema_version is not None:
current_version = models.get_schema_version()
if schema_version != current_version:
raise ValueError(
f"Unsupported schema version: {schema_version}; expected {current_version}"
)
start_time = datetime.now().astimezone()
metadata = get_metadata(nwb_path)
asset_md = prepare_metadata(metadata)
process_ndtypes(asset_md, metadata["nd_types"])
end_time = datetime.now().astimezone()
add_common_metadata(asset_md, nwb_path, start_time, end_time, digest)
asset_md["encodingFormat"] = "application/x-nwb"
asset_md["path"] = str(nwb_path)
return models.BareAsset(**asset_md)
def get_validation_errors(
self,
schema_version: Optional[str] = None,
devel_debug: bool = False,
) -> list[str]:
with open(self.filepath) as f:
meta = yaml_load(f, typ="safe")
if schema_version is None:
schema_version = meta.get("schemaVersion")
if schema_version is None:
return _check_required_fields(meta,
_required_dandiset_metadata_fields)
else:
current_version = get_schema_version()
if schema_version != current_version:
raise ValueError(
f"Unsupported schema version: {schema_version}; expected {current_version}"
)
try:
DandisetMeta(**meta)
except ValidationError as e:
if devel_debug:
raise
lgr.warning(
"Validation error for %s: %s",
self.filepath,
e,
extra={"validating": True},
)
return [str(e)]
except Exception as e:
if devel_debug:
raise
lgr.warning(
"Unexpected validation error for %s: %s",
self.filepath,
e,
extra={"validating": True},
)
return [f"Failed to initialize Dandiset meta: {e}"]
return []
def validate_dandiset_yaml(filepath, schema_version=None, devel_debug=False):
"""Validate dandiset.yaml"""
with open(filepath) as f:
meta = yaml_load(f, typ="safe")
if schema_version is None:
schema_version = meta.get("schemaVersion")
if schema_version is None:
return _check_required_fields(meta, _required_dandiset_metadata_fields)
else:
from dandischema.models import Dandiset as DandisetMeta
from dandischema.models import get_schema_version
from pydantic import ValidationError
current_version = get_schema_version()
if schema_version != current_version:
raise ValueError(
f"Unsupported schema version: {schema_version}; expected {current_version}"
)
try:
DandisetMeta(**meta)
except ValidationError as e:
if devel_debug:
raise
lgr.warning("Validation error for %s: %s",
filepath,
e,
extra={"validating": True})
return [str(e)]
except Exception as e:
if devel_debug:
raise
lgr.warning(
"Unexpected validation error for %s: %s",
filepath,
e,
extra={"validating": True},
)
return [f"Failed to initialize Dandiset meta: {e}"]
return []
def get_validation_errors(
self,
schema_version: Optional[str] = None,
devel_debug: bool = False,
) -> list[str]:
if schema_version is not None:
current_version = get_schema_version()
if schema_version != current_version:
raise ValueError(
f"Unsupported schema version: {schema_version}; expected {current_version}"
)
try:
asset = self.get_metadata(digest=DUMMY_DIGEST)
BareAsset(**asset.dict())
except ValidationError as e:
if devel_debug:
raise
lgr.warning(
"Validation error for %s: %s",
self.filepath,
e,
extra={"validating": True},
)
return [str(e)]
except Exception as e:
if devel_debug:
raise
lgr.warning(
"Unexpected validation error for %s: %s",
self.filepath,
e,
extra={"validating": True},
)
return [f"Failed to read metadata: {e}"]
return []
else:
# TODO: Do something else?
return []
def test_validate_simple1(simple1_nwb):
# this file should be ok
errors = validate_file(simple1_nwb, schema_version=get_schema_version())
assert not errors
def test_validate_simple1(simple1_nwb):
# this file should be ok
errors = dandi_file(simple1_nwb).get_validation_errors(
schema_version=get_schema_version())
assert not errors