def addAnnotators(self, study, params):
# TODO: make the loadmodel decorator use AccessType.WRITE,
# once permissions work
params = self._decodeParams(params)
self.requireParams(['userIds'], params)
creatorUser = self.getCurrentUser()
User().requireAdminStudy(creatorUser)
# Load all users before adding any, to ensure all are valid
if len(set(params['userIds'])) != len(params['userIds']):
raise ValidationException('Duplicate user IDs.', 'userIds')
annotatorUsers = [
User().load(userId, user=creatorUser, level=AccessType.READ, exc=True)
for userId in params['userIds']
]
# Existing duplicate Annotators are tricky to check for, because it's
# possible to have a Study with multiple annotator Users (each with a
# sub-Folder), but with no Images yet, and thus no Annotation (Items)
# inside yet
duplicateAnnotatorFolders = Folder().find({
'parentId': study['_id'],
'meta.userId': {'$in': [annotatorUser['_id'] for annotatorUser in annotatorUsers]}
})
if duplicateAnnotatorFolders.count():
# Just list the first duplicate
duplicateAnnotatorFolder = next(iter(duplicateAnnotatorFolders))
raise ValidationException('Annotator user "%s" is already part of the study.' %
duplicateAnnotatorFolder['meta']['userId'])
# Look up images only once for efficiency
images = Study().getImages(study)
for annotatorUser in annotatorUsers:
Study().addAnnotator(study, annotatorUser, creatorUser, images)
return self.getStudy(id=study['_id'], params={})
def postprocess(self, folder, skipJobs):
"""
Post-processing to be run after media/annotation import
When skipJobs=False, the following may run as jobs:
Transcoding of Video
Transcoding of Images
Conversion of KPF annotations into track JSON
In either case, the following may run synchronously:
Conversion of CSV annotations into track JSON
"""
user = self.getCurrentUser()
auxiliary = get_or_create_auxiliary_folder(folder, user)
if not skipJobs:
token = Token().createToken(user=user, days=1)
# transcode VIDEO if necessary
videoItems = Folder().childItems(
folder, filters={"lowerName": {
"$regex": videoRegex
}})
for item in videoItems:
convert_video.delay(
GetPathFromItemId(str(item["_id"])),
str(item["folderId"]),
auxiliary["_id"],
girder_job_title=(
"Converting {} to a web friendly format".format(
str(item["_id"]))),
girder_client_token=str(token["_id"]),
)
# transcode IMAGERY if necessary
imageItems = Folder().childItems(
folder, filters={"lowerName": {
"$regex": imageRegex
}})
safeImageItems = Folder().childItems(
folder, filters={"lowerName": {
"$regex": safeImageRegex
}})
if imageItems.count() > safeImageItems.count():
convert_images.delay(
folder["_id"],
girder_client_token=str(token["_id"]),
girder_job_title=
f"Converting {folder['_id']} to a web friendly format",
)
elif imageItems.count() > 0:
folder["meta"]["annotate"] = True
# transform KPF if necessary
ymlItems = Folder().childItems(
folder, filters={"lowerName": {
"$regex": ymlRegex
}})
if ymlItems.count() > 0:
# There might be up to 3 yamls
allFiles = [Item().childFiles(item)[0] for item in ymlItems]
saveTracks(folder,
meva_serializer.load_kpf_as_tracks(allFiles), user)
ymlItems.rewind()
for item in ymlItems:
Item().move(item, auxiliary)
Folder().save(folder)
# transform CSV if necessasry
csvItems = Folder().childItems(
folder,
filters={"lowerName": {
"$regex": csvRegex
}},
sort=[("created", pymongo.DESCENDING)],
)
if csvItems.count() >= 1:
file = Item().childFiles(csvItems.next())[0]
json_output = getTrackData(file)
saveTracks(folder, json_output, user)
csvItems.rewind()
for item in csvItems:
Item().move(item, auxiliary)
return folder
def postprocess(user: types.GirderUserModel, dsFolder: types.GirderModel,
skipJobs: bool) -> types.GirderModel:
"""
Post-processing to be run after media/annotation import
When skipJobs=False, the following may run as jobs:
Transcoding of Video
Transcoding of Images
Conversion of KPF annotations into track JSON
Extraction and upload of zip files
In either case, the following may run synchronously:
Conversion of CSV annotations into track JSON
"""
job_is_private = user.get(constants.UserPrivateQueueEnabledMarker, False)
isClone = fromMeta(dsFolder, constants.ForeignMediaIdMarker,
None) is not None
# add default confidence filter threshold to folder metadata
dsFolder['meta'][constants.ConfidenceFiltersMarker] = {'default': 0.1}
# Validate user-supplied metadata fields are present
if fromMeta(dsFolder, constants.FPSMarker) is None:
raise RestException(f'{constants.FPSMarker} missing from metadata')
if fromMeta(dsFolder, constants.TypeMarker) is None:
raise RestException(f'{constants.TypeMarker} missing from metadata')
if not skipJobs and not isClone:
token = Token().createToken(user=user, days=2)
# extract ZIP Files if not already completed
zipItems = list(Folder().childItems(
dsFolder,
filters={"lowerName": {
"$regex": constants.zipRegex
}},
))
if len(zipItems) > 1:
raise RestException('There are multiple zip files in the folder.')
for item in zipItems:
total_items = len(list((Folder().childItems(dsFolder))))
if total_items > 1:
raise RestException(
'There are multiple files besides a zip, cannot continue')
newjob = tasks.extract_zip.apply_async(
queue=_get_queue_name(user),
kwargs=dict(
folderId=str(item["folderId"]),
itemId=str(item["_id"]),
girder_job_title=
f"Extracting {item['_id']} to folder {str(dsFolder['_id'])}",
girder_client_token=str(token["_id"]),
girder_job_type="private" if job_is_private else "convert",
),
)
newjob.job[constants.JOBCONST_PRIVATE_QUEUE] = job_is_private
Job().save(newjob.job)
return dsFolder
# transcode VIDEO if necessary
videoItems = Folder().childItems(
dsFolder, filters={"lowerName": {
"$regex": constants.videoRegex
}})
for item in videoItems:
newjob = tasks.convert_video.apply_async(
queue=_get_queue_name(user),
kwargs=dict(
folderId=str(item["folderId"]),
itemId=str(item["_id"]),
girder_job_title=
f"Converting {item['_id']} to a web friendly format",
girder_client_token=str(token["_id"]),
girder_job_type="private" if job_is_private else "convert",
),
)
newjob.job[constants.JOBCONST_PRIVATE_QUEUE] = job_is_private
newjob.job[constants.JOBCONST_DATASET_ID] = dsFolder["_id"]
Job().save(newjob.job)
# transcode IMAGERY if necessary
imageItems = Folder().childItems(
dsFolder, filters={"lowerName": {
"$regex": constants.imageRegex
}})
safeImageItems = Folder().childItems(
dsFolder,
filters={"lowerName": {
"$regex": constants.safeImageRegex
}})
if imageItems.count() > safeImageItems.count():
newjob = tasks.convert_images.apply_async(
queue=_get_queue_name(user),
kwargs=dict(
folderId=dsFolder["_id"],
girder_client_token=str(token["_id"]),
girder_job_title=
f"Converting {dsFolder['_id']} to a web friendly format",
girder_job_type="private" if job_is_private else "convert",
),
)
newjob.job[constants.JOBCONST_PRIVATE_QUEUE] = job_is_private
newjob.job[constants.JOBCONST_DATASET_ID] = dsFolder["_id"]
Job().save(newjob.job)
elif imageItems.count() > 0:
dsFolder["meta"][constants.DatasetMarker] = True
# transform KPF if necessary
ymlItems = Folder().childItems(
dsFolder, filters={"lowerName": {
"$regex": constants.ymlRegex
}})
if ymlItems.count() > 0:
# There might be up to 3 yamls
def make_file_generator(item):
file = Item().childFiles(item)[0]
return File().download(file, headers=False)()
allFiles = [make_file_generator(item) for item in ymlItems]
data = meva.load_kpf_as_tracks(allFiles)
crud_annotation.save_annotations(dsFolder,
data.values(), [],
user,
overwrite=True,
description="Import from KPF")
ymlItems.rewind()
auxiliary = crud.get_or_create_auxiliary_folder(dsFolder, user)
for item in ymlItems:
Item().move(item, auxiliary)
Folder().save(dsFolder)
process_items(dsFolder, user)
return dsFolder