def test_draft_version_metadata_computed(draft_version: Version): original_metadata = {'schemaVersion': settings.DANDI_SCHEMA_VERSION} draft_version.metadata = original_metadata # Save the version to add computed properties to the metadata draft_version.save() expected_metadata = { **original_metadata, 'manifestLocation': [ f'{settings.DANDI_API_URL}/api/dandisets/{draft_version.dandiset.identifier}/versions/draft/assets/' # noqa: E501 ], 'name': draft_version.name, 'identifier': f'DANDI:{draft_version.dandiset.identifier}', 'version': draft_version.version, 'id': f'DANDI:{draft_version.dandiset.identifier}/{draft_version.version}', 'url': ( f'{settings.DANDI_WEB_APP_URL}/dandiset/' f'{draft_version.dandiset.identifier}/{draft_version.version}' ), 'repository': settings.DANDI_WEB_APP_URL, 'dateCreated': draft_version.dandiset.created.isoformat(), '@context': f'https://raw.githubusercontent.com/dandi/schema/master/releases/{settings.DANDI_SCHEMA_VERSION}/context.json', # noqa: E501 'assetsSummary': { 'numberOfBytes': 0, 'numberOfFiles': 0, 'schemaKey': 'AssetsSummary', }, } expected_metadata['citation'] = draft_version.citation(expected_metadata) assert draft_version.metadata == expected_metadata
def create_dev_dandiset(name: str, owner: str): owner = User.objects.get(email=owner) # Create a new dandiset dandiset = Dandiset() dandiset.save() dandiset.add_owner(owner) # Create the draft version version_metadata = { 'schemaVersion': settings.DANDI_SCHEMA_VERSION, 'schemaKey': 'Dandiset', 'description': 'An informative description', 'license': ['spdx:CC0-1.0'], 'contributor': [ { 'name': f'{owner.last_name}, {owner.first_name}', 'email': owner.email, 'roleName': ['dcite:ContactPerson'], 'schemaKey': 'Person', 'affiliation': [], 'includeInCitation': True, }, ], } draft_version = Version( dandiset=dandiset, name=name, metadata=version_metadata, version='draft', ) draft_version.save() uploaded_file = SimpleUploadedFile(name='foo/bar.txt', content=b'A' * 20) etag = '76d36e98f312e98ff908c8c82c8dd623-0' try: asset_blob = AssetBlob.objects.get(etag=etag) except AssetBlob.DoesNotExist: asset_blob = AssetBlob( blob_id=uuid4(), blob=uploaded_file, etag=etag, size=20, ) asset_blob.save() asset_metadata = { 'schemaVersion': settings.DANDI_SCHEMA_VERSION, 'encodingFormat': 'text/plain', 'schemaKey': 'Asset', } asset = Asset(blob=asset_blob, metadata=asset_metadata, path='foo/bar.txt') asset.save() draft_version.assets.add(asset) calculate_sha256(blob_id=asset_blob.blob_id) validate_asset_metadata(asset_id=asset.id) validate_version_metadata(version_id=draft_version.id)
def import_versions_from_response(api_url: str, version_api_response: dict, dandiset: Dandiset): """Import versions given a response from /api/dandisets/{identifier}/versions/.""" for result in version_api_response['results']: # get the metadata of this version metadata = requests.get( urljoin( api_url, f'/api/dandisets/{result["dandiset"]["identifier"]}/versions/{result["version"]}/', )).json() click.echo(f' Importing version "{result["version"]}"') version = Version( dandiset=dandiset, name=result['name'], version=result['version'], doi=result.get('doi'), status=Version.Status.PENDING, metadata=metadata, ) version.save() # Handle API pagination if version_api_response.get('next'): next_versions = requests.get(version_api_response.get('next')).json() import_versions_from_response(api_url, next_versions, dandiset)
def test_validate_version_metadata_malformed_schema_version( version: Version, asset: Asset): version.assets.add(asset) version.metadata['schemaVersion'] = 'xxx' version.save() tasks.validate_version_metadata(version.id) version.refresh_from_db() assert version.status == Version.Status.INVALID assert len(version.validation_errors) == 1 assert version.validation_errors[0]['message'].startswith( 'Metadata version xxx is not allowed.')
def create(self, request): serializer = VersionMetadataSerializer(data=request.data) serializer.is_valid(raise_exception=True) version_metadata, created = VersionMetadata.objects.get_or_create( name=serializer.validated_data['name'], metadata=serializer.validated_data['metadata'], ) if created: version_metadata.save() if 'identifier' in serializer.validated_data['metadata']: identifier = serializer.validated_data['metadata']['identifier'] if identifier.startswith('DANDI:'): identifier = identifier[6:] try: dandiset = Dandiset(id=int(identifier)) except ValueError: return Response(f'Invalid Identifier {identifier}', status=400) else: dandiset = Dandiset() try: # Without force_insert, Django will try to UPDATE an existing dandiset if one exists. # We want to throw an error if a dandiset already exists. dandiset.save(force_insert=True) except IntegrityError as e: # https://stackoverflow.com/questions/25368020/django-deduce-duplicate-key-exception-from-integrityerror # https://www.postgresql.org/docs/13/errcodes-appendix.html # Postgres error code 23505 == unique_violation if e.__cause__.pgcode == '23505': return Response( f'Dandiset {dandiset.identifier} Already Exists', status=400) raise e assign_perm('owner', request.user, dandiset) # Create new draft version version = Version(dandiset=dandiset, metadata=version_metadata, version='draft') version.save() serializer = DandisetDetailSerializer(instance=dandiset) return Response(serializer.data, status=status.HTTP_200_OK)
def test_validate_version_metadata_malformed_license(version: Version, asset: Asset): version.assets.add(asset) version.metadata['license'] = 'foo' version.save() tasks.validate_version_metadata(version.id) version.refresh_from_db() assert version.status == Version.Status.INVALID assert version.validation_errors == [{ 'field': 'license', 'message': "'foo' is not of type 'array'" }]
def test_validate_version_metadata_no_description(version: Version, asset: Asset): version.assets.add(asset) del version.metadata['description'] version.save() tasks.validate_version_metadata(version.id) version.refresh_from_db() assert version.status == Version.Status.INVALID assert version.validation_errors == [{ 'field': '', 'message': "'description' is a required property" }]
def test_published_version_metadata_computed(published_version: Version): original_metadata = {'schemaVersion': settings.DANDI_SCHEMA_VERSION} published_version.metadata = original_metadata # Save the version to add computed properties to the metadata published_version.save() expected_metadata = { **original_metadata, 'manifestLocation': [ ( f'http://{settings.MINIO_STORAGE_ENDPOINT}/test-dandiapi-dandisets' f'/test-prefix/dandisets/{published_version.dandiset.identifier}' f'/{published_version.version}/assets.yaml' ) ], 'name': published_version.name, 'identifier': f'DANDI:{published_version.dandiset.identifier}', 'version': published_version.version, 'id': f'DANDI:{published_version.dandiset.identifier}/{published_version.version}', 'doi': f'10.80507/dandi.{published_version.dandiset.identifier}/{published_version.version}', # noqa: E501 'url': ( f'{settings.DANDI_WEB_APP_URL}/dandiset/' f'{published_version.dandiset.identifier}/{published_version.version}' ), 'repository': settings.DANDI_WEB_APP_URL, 'dateCreated': published_version.dandiset.created.isoformat(), '@context': f'https://raw.githubusercontent.com/dandi/schema/master/releases/{settings.DANDI_SCHEMA_VERSION}/context.json', # noqa: E501 'assetsSummary': { 'numberOfBytes': 0, 'numberOfFiles': 0, 'schemaKey': 'AssetsSummary', }, } expected_metadata['citation'] = published_version.citation(expected_metadata) assert published_version.metadata == expected_metadata
def create(self, request: Request): """Create a new dandiset.""" serializer = VersionMetadataSerializer(data=request.data) serializer.is_valid(raise_exception=True) query_serializer = CreateDandisetQueryParameterSerializer( data=request.query_params) query_serializer.is_valid(raise_exception=True) if query_serializer.validated_data['embargo']: embargo_status = Dandiset.EmbargoStatus.EMBARGOED else: embargo_status = Dandiset.EmbargoStatus.OPEN name = serializer.validated_data['name'] metadata = serializer.validated_data['metadata'] # Strip away any computed fields metadata = Version.strip_metadata(metadata) # Only inject a schemaVersion and default contributor field if they are # not specified in the metadata metadata = { 'schemaKey': 'Dandiset', 'schemaVersion': settings.DANDI_SCHEMA_VERSION, 'contributor': [ { 'name': f'{request.user.last_name}, {request.user.first_name}', 'email': request.user.email, 'roleName': ['dcite:ContactPerson'], 'schemaKey': 'Person', 'affiliation': [], 'includeInCitation': True, }, ], # TODO: move this into dandischema 'access': [{ 'schemaKey': 'AccessRequirements', 'status': 'dandi:OpenAccess' if embargo_status == Dandiset.EmbargoStatus.OPEN else 'dandi:EmbargoedAccess', }], **metadata, } # Run the metadata through the pydantic model to automatically include any boilerplate # like the access or repository fields metadata = PydanticDandiset.unvalidated(**metadata).json_dict() if 'identifier' in serializer.validated_data['metadata']: identifier = serializer.validated_data['metadata']['identifier'] if identifier and not request.user.is_superuser: return Response( 'Creating a dandiset for a given identifier ' f'({identifier} was provided) is admin only operation.', status=403, ) if identifier.startswith('DANDI:'): identifier = identifier[6:] try: dandiset = Dandiset(id=int(identifier), embargo_status=embargo_status) except ValueError: return Response(f'Invalid Identifier {identifier}', status=400) else: dandiset = Dandiset(embargo_status=embargo_status) try: # Without force_insert, Django will try to UPDATE an existing dandiset if one exists. # We want to throw an error if a dandiset already exists. dandiset.save(force_insert=True) except IntegrityError as e: # https://stackoverflow.com/questions/25368020/django-deduce-duplicate-key-exception-from-integrityerror # https://www.postgresql.org/docs/13/errcodes-appendix.html # Postgres error code 23505 == unique_violation if e.__cause__.pgcode == '23505': return Response( f'Dandiset {dandiset.identifier} Already Exists', status=400) raise e logging.info( 'Created dandiset %s given request with name=%s and metadata=%s', dandiset.identifier, name, metadata, ) assign_perm('owner', request.user, dandiset) # Create new draft version version = Version( dandiset=dandiset, name=name, metadata=metadata, version='draft', status=Version.Status.PENDING, ) version.save() serializer = DandisetDetailSerializer(instance=dandiset) return Response(serializer.data, status=status.HTTP_200_OK)