def from_girder(cls, draft_folder_id: str, client: GirderClient) -> Dandiset:
"""
Return the Dandiset corresponding to a Girder `draft_folder_id`.
Creates the Dandiset if it does not exist.
"""
draft_folder = client.get_folder(draft_folder_id)
dandiset_identifier = draft_folder['name']
try:
dandiset_id = int(dandiset_identifier)
except ValueError:
raise GirderError(f'Invalid Dandiset identifier in Girder: {dandiset_identifier}')
try:
dandiset = Dandiset.objects.get(id=dandiset_id)
except ObjectDoesNotExist:
dandiset = Dandiset(id=dandiset_id, draft_folder_id=draft_folder_id)
dandiset.save()
from .draft_version import DraftVersion
draft = DraftVersion.from_girder_metadata(dandiset, draft_folder['meta'])
draft.full_clean()
draft.save()
else:
# If the Dandiset existed, sync the draft_folder_id
if dandiset.draft_folder_id != draft_folder_id:
raise GirderError(
f'Known Dandiset identifer {dandiset.identifier} does not'
f'match existing Girder folder id {dandiset.draft_folder_id}'
)
return dandiset
def sync(self, request):
if 'folder-id' not in request.query_params:
raise ValidationError('Missing query parameter "folder-id"')
draft_folder_id = request.query_params['folder-id']
with GirderClient() as client:
Dandiset.from_girder(draft_folder_id, client)
return Response('', status=status.HTTP_202_ACCEPTED)
def publish_version(dandiset_id: int) -> None:
with GirderClient(authenticate=True) as client:
dandiset = Dandiset.objects.get(pk=dandiset_id)
with client.dandiset_lock(dandiset.identifier):
version = Version.from_girder(dandiset, client)
for girder_file in client.files_in_folder(
dandiset.draft_folder_id):
Asset.from_girder(version, girder_file, client)
def from_girder(cls, dandiset: Dandiset, client: GirderClient) -> Version:
draft_folder = client.get_folder(dandiset.draft_folder_id)
metadata = draft_folder['meta']
version = cls.from_girder_metadata(dandiset, metadata)
version.full_clean()
version.save()
return version
def publish_version(dandiset_id: int, user_id) -> None:
dandiset = Dandiset.objects.get(pk=dandiset_id)
try:
with GirderClient(authenticate=True) as client:
with client.dandiset_lock(dandiset.identifier):
version = Version.from_girder(dandiset, client)
for girder_file in client.files_in_folder(dandiset.draft_folder_id):
Asset.from_girder(version, girder_file, client)
finally:
# The draft was locked in django by the publish action
# We need to unlock it now
dandiset.draft_version.unlock(User.objects.get(id=user_id))
def from_girder(cls, version: Version, girder_file: GirderFile,
client: GirderClient) -> Asset:
sha256_hasher = hashlib.sha256()
blob_size = 0
with NamedTemporaryFile('r+b') as local_stream:
logger.info(f'Downloading file {girder_file.girder_id}')
with client.iter_file_content(
girder_file.girder_id) as file_content_iter:
for chunk in file_content_iter:
sha256_hasher.update(chunk)
blob_size += len(chunk)
local_stream.write(chunk)
logger.info(f'Downloaded file {girder_file.girder_id}')
local_stream.seek(0)
# local_path = Path(local_stream.name)
sha256 = sha256_hasher.hexdigest()
blob = File(file=local_stream, name=girder_file.path.lstrip('/'))
# content_type is not part of the base File class (it on some other subclasses),
# but regardless S3Boto3Storage will respect and use it, if it's set
blob.content_type = 'application/octet-stream'
asset = Asset(
version=version,
path=girder_file.path,
size=blob_size,
sha256=sha256,
metadata=girder_file.metadata,
blob=blob,
)
# The actual upload of blob occurs when the asset is saved
asset.save()
return asset