def nwb2asset( nwb_path: Union[str, Path], digest: Optional[Digest] = None, schema_version: Optional[str] = None, ) -> models.BareAsset: if schema_version is not None: current_version = models.get_schema_version() if schema_version != current_version: raise ValueError( f"Unsupported schema version: {schema_version}; expected {current_version}" ) start_time = datetime.now().astimezone() metadata = get_metadata(nwb_path) if digest is not None: metadata["digest"] = digest.value metadata["digest_type"] = digest.algorithm.name metadata["contentSize"] = op.getsize(nwb_path) metadata["encodingFormat"] = "application/x-nwb" metadata["dateModified"] = get_utcnow_datetime() metadata["blobDateModified"] = ensure_datetime(os.stat(nwb_path).st_mtime) metadata["path"] = str(nwb_path) if metadata["blobDateModified"] > metadata["dateModified"]: lgr.warning("mtime %s of %s is in the future", metadata["blobDateModified"], nwb_path) asset = metadata2asset(metadata) asset = process_ndtypes(asset, metadata["nd_types"]) end_time = datetime.now().astimezone() if asset.wasGeneratedBy is None: asset.wasGeneratedBy = [] asset.wasGeneratedBy.append(get_generator(start_time, end_time)) return asset
def test_check_schema_version_matches_default() -> None: responses.add( responses.GET, "https://test.nil/api/info/", json={"schema_version": get_schema_version()}, ) client = DandiAPIClient("https://test.nil/api") client.check_schema_version()
def __init__(self, path, allow_empty=False, schema_version=None): if schema_version is not None: current_version = get_schema_version() if schema_version != current_version: raise ValueError( f"Unsupported schema version: {schema_version}; expected {current_version}" ) super().__init__(path, allow_empty=allow_empty)
def validate_dandi_nwb(filepath, schema_version=None, devel_debug=False): """Provide validation of .nwb file regarding requirements we impose""" if schema_version is not None: from dandischema.models import BareAsset, get_schema_version from pydantic import ValidationError from .metadata import nwb2asset current_version = get_schema_version() if schema_version != current_version: raise ValueError( f"Unsupported schema version: {schema_version}; expected {current_version}" ) try: asset = nwb2asset(filepath, digest=32 * "d" + "-1", digest_type="dandi_etag") BareAsset(**asset.dict()) except ValidationError as e: if devel_debug: raise lgr.warning("Validation error for %s: %s", filepath, e, extra={"validating": True}) return [str(e)] except Exception as e: if devel_debug: raise lgr.warning( "Unexpected validation error for %s: %s", filepath, e, extra={"validating": True}, ) return [f"Failed to read metadata: {e}"] return [] else: # make sure that we have some basic metadata fields we require try: meta = get_metadata(filepath) except Exception as e: if devel_debug: raise lgr.warning( "Failed to read metadata in %s: %s", filepath, e, extra={"validating": True}, ) return [f"Failed to read metadata: {e}"] return _check_required_fields(meta, _required_nwb_metadata_fields)
def __init__( self, path: Union[str, Path], allow_empty: bool = False, schema_version: Optional[str] = None, ) -> None: if schema_version is not None: current_version = get_schema_version() if schema_version != current_version: raise ValueError( f"Unsupported schema version: {schema_version}; expected {current_version}" ) super().__init__(path, allow_empty=allow_empty)
def nwb2asset( nwb_path: Union[str, Path], digest: Optional[Digest] = None, schema_version: Optional[str] = None, ) -> models.BareAsset: if schema_version is not None: current_version = models.get_schema_version() if schema_version != current_version: raise ValueError( f"Unsupported schema version: {schema_version}; expected {current_version}" ) start_time = datetime.now().astimezone() metadata = get_metadata(nwb_path) asset_md = prepare_metadata(metadata) process_ndtypes(asset_md, metadata["nd_types"]) end_time = datetime.now().astimezone() add_common_metadata(asset_md, nwb_path, start_time, end_time, digest) asset_md["encodingFormat"] = "application/x-nwb" asset_md["path"] = str(nwb_path) return models.BareAsset(**asset_md)
def get_validation_errors( self, schema_version: Optional[str] = None, devel_debug: bool = False, ) -> list[str]: with open(self.filepath) as f: meta = yaml_load(f, typ="safe") if schema_version is None: schema_version = meta.get("schemaVersion") if schema_version is None: return _check_required_fields(meta, _required_dandiset_metadata_fields) else: current_version = get_schema_version() if schema_version != current_version: raise ValueError( f"Unsupported schema version: {schema_version}; expected {current_version}" ) try: DandisetMeta(**meta) except ValidationError as e: if devel_debug: raise lgr.warning( "Validation error for %s: %s", self.filepath, e, extra={"validating": True}, ) return [str(e)] except Exception as e: if devel_debug: raise lgr.warning( "Unexpected validation error for %s: %s", self.filepath, e, extra={"validating": True}, ) return [f"Failed to initialize Dandiset meta: {e}"] return []
def validate_dandiset_yaml(filepath, schema_version=None, devel_debug=False): """Validate dandiset.yaml""" with open(filepath) as f: meta = yaml_load(f, typ="safe") if schema_version is None: schema_version = meta.get("schemaVersion") if schema_version is None: return _check_required_fields(meta, _required_dandiset_metadata_fields) else: from dandischema.models import Dandiset as DandisetMeta from dandischema.models import get_schema_version from pydantic import ValidationError current_version = get_schema_version() if schema_version != current_version: raise ValueError( f"Unsupported schema version: {schema_version}; expected {current_version}" ) try: DandisetMeta(**meta) except ValidationError as e: if devel_debug: raise lgr.warning("Validation error for %s: %s", filepath, e, extra={"validating": True}) return [str(e)] except Exception as e: if devel_debug: raise lgr.warning( "Unexpected validation error for %s: %s", filepath, e, extra={"validating": True}, ) return [f"Failed to initialize Dandiset meta: {e}"] return []
def get_validation_errors( self, schema_version: Optional[str] = None, devel_debug: bool = False, ) -> list[str]: if schema_version is not None: current_version = get_schema_version() if schema_version != current_version: raise ValueError( f"Unsupported schema version: {schema_version}; expected {current_version}" ) try: asset = self.get_metadata(digest=DUMMY_DIGEST) BareAsset(**asset.dict()) except ValidationError as e: if devel_debug: raise lgr.warning( "Validation error for %s: %s", self.filepath, e, extra={"validating": True}, ) return [str(e)] except Exception as e: if devel_debug: raise lgr.warning( "Unexpected validation error for %s: %s", self.filepath, e, extra={"validating": True}, ) return [f"Failed to read metadata: {e}"] return [] else: # TODO: Do something else? return []
def test_validate_simple1(simple1_nwb): # this file should be ok errors = validate_file(simple1_nwb, schema_version=get_schema_version()) assert not errors
def test_validate_simple1(simple1_nwb): # this file should be ok errors = dandi_file(simple1_nwb).get_validation_errors( schema_version=get_schema_version()) assert not errors