def postprocess(self, folder, skipJobs):
        """
        Post-processing to be run after media/annotation import


        When skipJobs=False, the following may run as jobs:
            Transcoding of Video
            Transcoding of Images
            Conversion of KPF annotations into track JSON

        In either case, the following may run synchronously:
            Conversion of CSV annotations into track JSON
        """
        user = self.getCurrentUser()
        auxiliary = get_or_create_auxiliary_folder(folder, user)

        if not skipJobs:
            token = Token().createToken(user=user, days=1)
            # transcode VIDEO if necessary
            videoItems = Folder().childItems(
                folder, filters={"lowerName": {
                    "$regex": videoRegex
                }})

            for item in videoItems:
                convert_video.delay(
                    GetPathFromItemId(str(item["_id"])),
                    str(item["folderId"]),
                    auxiliary["_id"],
                    girder_job_title=(
                        "Converting {} to a web friendly format".format(
                            str(item["_id"]))),
                    girder_client_token=str(token["_id"]),
                )

            # transcode IMAGERY if necessary
            imageItems = Folder().childItems(
                folder, filters={"lowerName": {
                    "$regex": imageRegex
                }})
            safeImageItems = Folder().childItems(
                folder, filters={"lowerName": {
                    "$regex": safeImageRegex
                }})

            if imageItems.count() > safeImageItems.count():
                convert_images.delay(
                    folder["_id"],
                    girder_client_token=str(token["_id"]),
                    girder_job_title=
                    f"Converting {folder['_id']} to a web friendly format",
                )
            elif imageItems.count() > 0:
                folder["meta"]["annotate"] = True

            # transform KPF if necessary
            ymlItems = Folder().childItems(
                folder, filters={"lowerName": {
                    "$regex": ymlRegex
                }})
            if ymlItems.count() > 0:
                # There might be up to 3 yamls
                allFiles = [Item().childFiles(item)[0] for item in ymlItems]
                saveTracks(folder,
                           meva_serializer.load_kpf_as_tracks(allFiles), user)
                ymlItems.rewind()
                for item in ymlItems:
                    Item().move(item, auxiliary)

            Folder().save(folder)

        # transform CSV if necessasry
        csvItems = Folder().childItems(
            folder,
            filters={"lowerName": {
                "$regex": csvRegex
            }},
            sort=[("created", pymongo.DESCENDING)],
        )
        if csvItems.count() >= 1:
            file = Item().childFiles(csvItems.next())[0]
            json_output = getTrackData(file)
            saveTracks(folder, json_output, user)
            csvItems.rewind()
            for item in csvItems:
                Item().move(item, auxiliary)

        return folder
Beispiel #2
0
def postprocess(user: types.GirderUserModel, dsFolder: types.GirderModel,
                skipJobs: bool) -> types.GirderModel:
    """
    Post-processing to be run after media/annotation import

    When skipJobs=False, the following may run as jobs:
        Transcoding of Video
        Transcoding of Images
        Conversion of KPF annotations into track JSON
        Extraction and upload of zip files

    In either case, the following may run synchronously:
        Conversion of CSV annotations into track JSON
    """
    job_is_private = user.get(constants.UserPrivateQueueEnabledMarker, False)
    isClone = fromMeta(dsFolder, constants.ForeignMediaIdMarker,
                       None) is not None
    # add default confidence filter threshold to folder metadata
    dsFolder['meta'][constants.ConfidenceFiltersMarker] = {'default': 0.1}

    # Validate user-supplied metadata fields are present
    if fromMeta(dsFolder, constants.FPSMarker) is None:
        raise RestException(f'{constants.FPSMarker} missing from metadata')
    if fromMeta(dsFolder, constants.TypeMarker) is None:
        raise RestException(f'{constants.TypeMarker} missing from metadata')

    if not skipJobs and not isClone:
        token = Token().createToken(user=user, days=2)

        # extract ZIP Files if not already completed
        zipItems = list(Folder().childItems(
            dsFolder,
            filters={"lowerName": {
                "$regex": constants.zipRegex
            }},
        ))
        if len(zipItems) > 1:
            raise RestException('There are multiple zip files in the folder.')
        for item in zipItems:
            total_items = len(list((Folder().childItems(dsFolder))))
            if total_items > 1:
                raise RestException(
                    'There are multiple files besides a zip, cannot continue')
            newjob = tasks.extract_zip.apply_async(
                queue=_get_queue_name(user),
                kwargs=dict(
                    folderId=str(item["folderId"]),
                    itemId=str(item["_id"]),
                    girder_job_title=
                    f"Extracting {item['_id']} to folder {str(dsFolder['_id'])}",
                    girder_client_token=str(token["_id"]),
                    girder_job_type="private" if job_is_private else "convert",
                ),
            )
            newjob.job[constants.JOBCONST_PRIVATE_QUEUE] = job_is_private
            Job().save(newjob.job)
            return dsFolder

        # transcode VIDEO if necessary
        videoItems = Folder().childItems(
            dsFolder, filters={"lowerName": {
                "$regex": constants.videoRegex
            }})

        for item in videoItems:
            newjob = tasks.convert_video.apply_async(
                queue=_get_queue_name(user),
                kwargs=dict(
                    folderId=str(item["folderId"]),
                    itemId=str(item["_id"]),
                    girder_job_title=
                    f"Converting {item['_id']} to a web friendly format",
                    girder_client_token=str(token["_id"]),
                    girder_job_type="private" if job_is_private else "convert",
                ),
            )
            newjob.job[constants.JOBCONST_PRIVATE_QUEUE] = job_is_private
            newjob.job[constants.JOBCONST_DATASET_ID] = dsFolder["_id"]
            Job().save(newjob.job)

        # transcode IMAGERY if necessary
        imageItems = Folder().childItems(
            dsFolder, filters={"lowerName": {
                "$regex": constants.imageRegex
            }})
        safeImageItems = Folder().childItems(
            dsFolder,
            filters={"lowerName": {
                "$regex": constants.safeImageRegex
            }})

        if imageItems.count() > safeImageItems.count():
            newjob = tasks.convert_images.apply_async(
                queue=_get_queue_name(user),
                kwargs=dict(
                    folderId=dsFolder["_id"],
                    girder_client_token=str(token["_id"]),
                    girder_job_title=
                    f"Converting {dsFolder['_id']} to a web friendly format",
                    girder_job_type="private" if job_is_private else "convert",
                ),
            )
            newjob.job[constants.JOBCONST_PRIVATE_QUEUE] = job_is_private
            newjob.job[constants.JOBCONST_DATASET_ID] = dsFolder["_id"]
            Job().save(newjob.job)

        elif imageItems.count() > 0:
            dsFolder["meta"][constants.DatasetMarker] = True

        # transform KPF if necessary
        ymlItems = Folder().childItems(
            dsFolder, filters={"lowerName": {
                "$regex": constants.ymlRegex
            }})
        if ymlItems.count() > 0:
            # There might be up to 3 yamls
            def make_file_generator(item):
                file = Item().childFiles(item)[0]
                return File().download(file, headers=False)()

            allFiles = [make_file_generator(item) for item in ymlItems]
            data = meva.load_kpf_as_tracks(allFiles)
            crud_annotation.save_annotations(dsFolder,
                                             data.values(), [],
                                             user,
                                             overwrite=True,
                                             description="Import from KPF")
            ymlItems.rewind()
            auxiliary = crud.get_or_create_auxiliary_folder(dsFolder, user)
            for item in ymlItems:
                Item().move(item, auxiliary)

        Folder().save(dsFolder)

    process_items(dsFolder, user)
    return dsFolder