예제 #1
0
    def update_source_related_channels(channel, experiment, source_channels, related_channels):
        """
        Update a list of source and related channels

        Args:
            related_channels: New list of related channels
            source_channels: New list of source channels
            experiment: Experiment for the current channel
            channel: Curren channel

        Returns:
            Updated Channel

        """
        try:
            # update ist of sources
            # Get all the source
            cur_sources = channel.sources.all()
            # Get the list of sources to remove
            rm_sources = [ch for ch in cur_sources if ch not in source_channels]
            for source in rm_sources:
                channel.remove_source(source)

            # add new sources
            add_sources = [ch for ch in source_channels if ch not in cur_sources]
            for source_channel in add_sources:
                channel.add_source(source_channel)

            cur_related = channel.related.all()
            rm_related = [ch for ch in cur_related if ch not in related_channels]
            for related in rm_related:
                channel.related.remove(related)

            add_related = [ch for ch in related_channels if ch not in cur_related]
            for related_channel in add_related:
                channel.related.add(related_channel.pk)

            channel.save()
            return channel
        except Exception as err:
            channel.delete()
            raise BossError("Exception adding source/related channels.{}".format(err), ErrorCodes.INVALID_POST_ARGUMENT)
예제 #2
0
    def get_ingest_job(self, ingest_job_id):
        """
        Get the ingest job with the specific id
        Args:
            ingest_job_id: Id of the ingest job

        Returns:
            IngestJob : Data model with the ingest job if the id is valid

        Raises:
            BossError : If the ingets job id does not exist

        """
        try:
            ingest_job = IngestJob.objects.get(id=ingest_job_id)
            return ingest_job
        except IngestJob.DoesNotExist:
            raise BossError(
                "The ingest job with id {} does not exist".format(
                    str(ingest_job_id)), ErrorCodes.OBJECT_NOT_FOUND)
예제 #3
0
def enqueue_job(session, args, downsample_sqs):
    """Enqueue downsample job

    Args:
        session (boto3.session):
        args (dict): Arguments passed to the downsample step function via SQS
        downsample_sqs (str): URL of SQS queue

    Raises:
        (BossError): If failed to enqueue job.
    """
    rows_updated = (Channel.objects.filter(id=args['channel_id']).exclude(
        downsample_status=Channel.DownsampleStatus.IN_PROGRESS).exclude(
            downsample_status=Channel.DownsampleStatus.QUEUED).update(
                downsample_status=Channel.DownsampleStatus.QUEUED))
    if rows_updated == 0:
        raise BossError(DOWNSAMPLE_CANNOT_BE_QUEUED_ERR_MSG,
                        ErrorCodes.BAD_REQUEST)

    _sqs_enqueue(session, args, downsample_sqs)
예제 #4
0
    def _convert_string_to_ingest_job(self, s):
        """
        Convert a string representation of ingest_type to int.

        Args:
            s (str):

        Returns:
            (int): IngestJob.TILE_INGEST | IngestJob.VOLUMETRIC_INGEST

        Raises:
            (BossError): If string is invalid.
        """
        lowered = s.lower()
        if lowered == 'tile':
            return IngestJob.TILE_INGEST
        if lowered == 'volumetric':
            return IngestJob.VOLUMETRIC_INGEST

        raise BossError('Unknown ingest_type: {}'.format(s))
예제 #5
0
    def create_ingest_job(self):
        """

        Returns:

        """

        ingest_job_serializer_data = {
            'creator': self.owner,
            'collection': self.collection.name,
            'experiment': self.experiment.name,
            'channel_layer': self.channel_layer.name,
            'config_data': json.dumps(self.config.config_data),
            'resolution': self.resolution,
            'x_start': self.config.config_data["ingest_job"]["extent"]["x"][0],
            'x_stop': self.config.config_data["ingest_job"]["extent"]["x"][1],
            'y_start': self.config.config_data["ingest_job"]["extent"]["y"][0],
            'y_stop': self.config.config_data["ingest_job"]["extent"]["y"][1],
            'z_start': self.config.config_data["ingest_job"]["extent"]["z"][0],
            'z_stop': self.config.config_data["ingest_job"]["extent"]["z"][1],
            't_start': self.config.config_data["ingest_job"]["extent"]["t"][0],
            't_stop': self.config.config_data["ingest_job"]["extent"]["t"][1],
            'tile_size_x':
            self.config.config_data["ingest_job"]["tile_size"]["x"],
            'tile_size_y':
            self.config.config_data["ingest_job"]["tile_size"]["y"],
            'tile_size_z':
            self.config.config_data["ingest_job"]["tile_size"]["z"],
            'tile_size_t':
            self.config.config_data["ingest_job"]["tile_size"]["t"],
        }
        serializer = IngestJobCreateSerializer(data=ingest_job_serializer_data)
        if serializer.is_valid():
            ingest_job = serializer.save()
            return ingest_job

        else:
            raise BossError("{}".format(serializer.errors),
                            ErrorCodes.SERIALIZATION_ERROR)
예제 #6
0
    def test_complete_should_fail_if_queue_not_empty(self, ingest_mgr_creator):
        job_id = 28
        ingest_job = MagicMock(spec=IngestJob)
        ingest_job.status = IngestJob.UPLOADING
        fake_ingest_mgr = MagicMock(spec=IngestManager)
        fake_ingest_mgr.get_ingest_job.return_value = ingest_job
        # This method tries to move from UPLOADING to WAIT_ON_QUEUES.
        fake_ingest_mgr.try_enter_wait_on_queue_state.side_effect = BossError(
            INGEST_QUEUE_NOT_EMPTY_ERR_MSG, ErrorCodes.BAD_REQUEST)

        # Provide the fake when the complete view instantiates an IngestManager.
        ingest_mgr_creator.return_value = fake_ingest_mgr

        testuser = User.objects.create_user(username='******')
        ingest_job.creator = testuser
        self.client.force_authenticate(user=testuser)

        url = '/{}/ingest/{}/complete'.format(version, job_id)
        resp = self.client.post(url, format='json')
        self.assertEqual(400, resp.status_code)
        actual = resp.json()
        self.assertIn('wait_secs', actual)
        self.assertIn('info', actual)
예제 #7
0
    def delete_tiles(self, ingest_job):
        """
        Delete all remaining tiles from the tile index database and tile bucket

        5/24/2018 - This code depends on a GSI for the tile index.  The GSI was
        removed because its key didn't shard well.  Cleanup will now be handled
        by TTL policies applied to the tile bucket and the tile index.  This
        method will be removed once that code is merged.

        Args:
            ingest_job: Ingest job model

        Returns:
            None
        Raises:
            BossError : For exceptions that happen while deleting the tiles and index

        """
        try:
            # Get all the chunks for a job
            tiledb = BossTileIndexDB(ingest_job.collection + '&' +
                                     ingest_job.experiment)
            tilebucket = TileBucket(ingest_job.collection + '&' +
                                    ingest_job.experiment)
            chunks = list(tiledb.getTaskItems(ingest_job.id))

            for chunk in chunks:
                # delete each tile in the chunk
                for key in chunk['tile_uploaded_map']:
                    response = tilebucket.deleteObject(key)
                tiledb.deleteCuboid(chunk['chunk_key'], ingest_job.id)

        except Exception as e:
            raise BossError(
                "Exception while deleteing tiles for the ingest job {}. {}".
                format(ingest_job.id, e), ErrorCodes.BOSS_SYSTEM_ERROR)
예제 #8
0
    def create_ingest_job(self):
        """
        Create a new ingest job using the parameters in the ingest config data file

        Returns:
            IngestJob : Data model with the current ingest job

        Raises:
            BossError : For serialization errors that occur while creating a ingest job or if ingest_type is invalid
        """

        ingest_job_serializer_data = {
            'creator': self.owner,
            'collection': self.collection.name,
            'experiment': self.experiment.name,
            'channel': self.channel.name,
            'config_data': json.dumps(self.config.config_data),
            'resolution': self.resolution,
            'x_start': self.config.config_data["ingest_job"]["extent"]["x"][0],
            'x_stop': self.config.config_data["ingest_job"]["extent"]["x"][1],
            'y_start': self.config.config_data["ingest_job"]["extent"]["y"][0],
            'y_stop': self.config.config_data["ingest_job"]["extent"]["y"][1],
            'z_start': self.config.config_data["ingest_job"]["extent"]["z"][0],
            'z_stop': self.config.config_data["ingest_job"]["extent"]["z"][1],
            't_start': self.config.config_data["ingest_job"]["extent"]["t"][0],
            't_stop': self.config.config_data["ingest_job"]["extent"]["t"][1],
        }

        if "ingest_type" in self.config.config_data["ingest_job"]:
            ingest_job_serializer_data[
                "ingest_type"] = self._convert_string_to_ingest_job(
                    self.config.config_data["ingest_job"]["ingest_type"])
        else:
            ingest_job_serializer_data["ingest_type"] = IngestJob.TILE_INGEST

        if ingest_job_serializer_data["ingest_type"] == IngestJob.TILE_INGEST:
            ingest_job_serializer_data[
                'tile_size_x'] = self.config.config_data["ingest_job"][
                    "tile_size"]["x"]
            ingest_job_serializer_data[
                'tile_size_y'] = self.config.config_data["ingest_job"][
                    "tile_size"]["y"]
            #ingest_job_serializer_data['tile_size_z'] = self.config.config_data["ingest_job"]["tile_size"]["z"]
            ingest_job_serializer_data['tile_size_z'] = 1
            ingest_job_serializer_data[
                'tile_size_t'] = self.config.config_data["ingest_job"][
                    "tile_size"]["t"]
        elif ingest_job_serializer_data[
                "ingest_type"] == IngestJob.VOLUMETRIC_INGEST:
            ingest_job_serializer_data[
                'tile_size_x'] = self.config.config_data["ingest_job"][
                    "chunk_size"]["x"]
            ingest_job_serializer_data[
                'tile_size_y'] = self.config.config_data["ingest_job"][
                    "chunk_size"]["y"]
            ingest_job_serializer_data[
                'tile_size_z'] = self.config.config_data["ingest_job"][
                    "chunk_size"]["z"]
            ingest_job_serializer_data['tile_size_t'] = 1
        else:
            raise BossError(
                'Invalid ingest_type: {}'.format(
                    ingest_job_serializer_data["ingest_type"]),
                ErrorCodes.UNABLE_TO_VALIDATE)

        serializer = IngestJobCreateSerializer(data=ingest_job_serializer_data)
        if serializer.is_valid():
            ingest_job = serializer.save()
            return ingest_job

        else:
            raise BossError("{}".format(serializer.errors),
                            ErrorCodes.SERIALIZATION_ERROR)
예제 #9
0
    def get(self, request, ingest_job_id=None):
        """
        Join a job with the specified job id or list all job ids if ingest_job_id is omitted
        Args:
            request: Django rest framework request object
            ingest_job_id: Ingest job id

        Returns:
            Ingest job
        """
        try:
            if ingest_job_id is None:
                # If the job ID is empty on a get, you are listing jobs
                return self.list_ingest_jobs(request)

            ingest_mgmr = IngestManager()
            ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)

            # Check permissions
            if not self.is_user_or_admin(request, ingest_job):
                return BossHTTPError(
                    "Only the creator or admin can join an ingest job",
                    ErrorCodes.INGEST_NOT_CREATOR)

            serializer = IngestJobListSerializer(ingest_job)

            # Start setting up output
            data = {'ingest_job': serializer.data}

            if ingest_job.status == 3:
                # The job has been deleted
                raise BossError(
                    "The job with id {} has been deleted".format(
                        ingest_job_id), ErrorCodes.INVALID_REQUEST)
            elif ingest_job.status == 2 or ingest_job.status == 4:
                # Failed job or completed job
                return Response(data, status=status.HTTP_200_OK)

            elif ingest_job.status == 0:
                # Job is still in progress
                # check status of the step function
                session = bossutils.aws.get_session()
                if bossutils.aws.sfn_status(
                        session, ingest_job.step_function_arn) == 'SUCCEEDED':
                    # generate credentials
                    ingest_job.status = 1
                    ingest_job.save()
                    ingest_mgmr.generate_ingest_credentials(ingest_job)
                elif bossutils.aws.sfn_status(
                        session, ingest_job.step_function_arn) == 'FAILED':
                    # This indicates an error in step function
                    raise BossError(
                        "Error generating ingest job messages"
                        " Delete the ingest job with id {} and try again.".
                        format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)

            if ingest_job.status == 1:
                data['ingest_job']['status'] = 1
                ingest_creds = IngestCredentials()
                data['credentials'] = ingest_creds.get_credentials(
                    ingest_job.id)
            else:
                data['credentials'] = None

            data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
            data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
            data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
            data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG

            # add the lambda - Possibly remove this later
            config = bossutils.configuration.BossConfig()
            data['ingest_lambda'] = config["lambda"]["page_in_function"]

            # Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly
            collection = Collection.objects.get(
                name=data['ingest_job']["collection"])
            experiment = Experiment.objects.get(
                name=data['ingest_job']["experiment"], collection=collection)
            coord_frame = experiment.coord_frame
            channel = Channel.objects.get(name=data['ingest_job']["channel"],
                                          experiment=experiment)

            resource = {}
            resource['boss_key'] = '{}&{}&{}'.format(
                data['ingest_job']["collection"],
                data['ingest_job']["experiment"],
                data['ingest_job']["channel"])
            resource['lookup_key'] = '{}&{}&{}'.format(collection.id,
                                                       experiment.id,
                                                       channel.id)

            # The Lambda function needs certain resource properties to perform write ops. Set required things only.
            # This is because S3 metadata is limited to 2kb, so we only set the bits of info needed, and in the lambda
            # Function Populate the rest with dummy info
            # IF YOU NEED ADDITIONAL DATA YOU MUST ADD IT HERE AND IN THE LAMBDA FUNCTION
            resource['channel'] = {}
            resource['channel']['type'] = channel.type
            resource['channel']['datatype'] = channel.datatype
            resource['channel']['base_resolution'] = channel.base_resolution

            resource['experiment'] = {}
            resource['experiment'][
                'num_hierarchy_levels'] = experiment.num_hierarchy_levels
            resource['experiment'][
                'hierarchy_method'] = experiment.hierarchy_method

            resource['coord_frame'] = {}
            resource['coord_frame']['x_voxel_size'] = coord_frame.x_voxel_size
            resource['coord_frame']['y_voxel_size'] = coord_frame.y_voxel_size
            resource['coord_frame']['z_voxel_size'] = coord_frame.z_voxel_size

            # Set resource
            data['resource'] = resource

            return Response(data, status=status.HTTP_200_OK)
        except BossError as err:
            return err.to_http()
        except Exception as err:
            return BossError("{}".format(err),
                             ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
예제 #10
0
    def post(self, request, ingest_job_id):
        """
        Signal an ingest job is complete and should be cleaned up by POSTing to this view

        Args:
            request: Django Rest framework Request object
            ingest_job_id: Ingest job id

        Returns:


        """
        try:
            blog = BossLogger().logger
            ingest_mgmr = IngestManager()
            ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)

            if ingest_job.status == IngestJob.PREPARING:
                # If status is Preparing. Deny
                return BossHTTPError(
                    "You cannot complete a job that is still preparing. You must cancel instead.",
                    ErrorCodes.BAD_REQUEST)
            elif ingest_job.status == IngestJob.UPLOADING:
                # Check if user is the ingest job creator or the sys admin
                if not self.is_user_or_admin(request, ingest_job):
                    return BossHTTPError(
                        "Only the creator or admin can start verification of an ingest job",
                        ErrorCodes.INGEST_NOT_CREATOR)

                # Disable verification until it is reworked and always return
                # success for now.
                blog.info(
                    'Telling client job complete - completion/verificcation to be fixed later.'
                )
                return Response(status=status.HTTP_204_NO_CONTENT)
                """
                blog.info('Verifying ingest job {}'.format(ingest_job_id))

                # Start verification process
                if not ingest_mgmr.verify_ingest_job(ingest_job):
                    # Ingest not finished
                    return Response(status=status.HTTP_202_ACCEPTED)
                """

                # Verification successful, fall through to the complete process.

            elif ingest_job.status == IngestJob.COMPLETE:
                # If status is already Complete, just return another 204
                return Response(status=status.HTTP_204_NO_CONTENT)
            elif ingest_job.status == IngestJob.DELETED:
                # Job had already been cancelled
                return BossHTTPError("Ingest job has already been cancelled.",
                                     ErrorCodes.BAD_REQUEST)
            elif ingest_job.status == IngestJob.FAILED:
                # Job had failed
                return BossHTTPError(
                    "Ingest job has failed during creation. You must Cancel instead.",
                    ErrorCodes.BAD_REQUEST)

            # Complete the job.
            blog.info("Completing Ingest Job {}".format(ingest_job_id))

            # Check if user is the ingest job creator or the sys admin
            if not self.is_user_or_admin(request, ingest_job):
                return BossHTTPError(
                    "Only the creator or admin can complete an ingest job",
                    ErrorCodes.INGEST_NOT_CREATOR)

            # TODO SH This is a quick fix to make sure the ingest-client does not run close option.
            #      the clean up code commented out below, because it is not working correctly.
            return Response(status=status.HTTP_204_NO_CONTENT)

            # if ingest_job.ingest_type == IngestJob.TILE_INGEST:
            #     # Check if any messages remain in the ingest queue
            #     ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(ingest_job)
            #     num_messages_in_queue = int(ingest_queue.queue.attributes['ApproximateNumberOfMessages'])
            #
            #     # Kick off extra lambdas just in case
            #     if num_messages_in_queue:
            #         blog.info("{} messages remaining in Ingest Queue".format(num_messages_in_queue))
            #         ingest_mgmr.invoke_ingest_lambda(ingest_job, num_messages_in_queue)
            #
            #         # Give lambda a few seconds to fire things off
            #         time.sleep(30)
            #
            #     ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
            #
            # elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
            #     ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
            #
            # # ToDo: call cleanup method for volumetric ingests.  Don't want
            # # to cleanup until after testing with real data.
            # #ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
            #
            # blog.info("Complete successful")
            # return Response(status=status.HTTP_204_NO_CONTENT)
        except BossError as err:
            return err.to_http()
        except Exception as err:
            blog.error('Caught general exception: {}'.format(err))
            return BossError("{}".format(err),
                             ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
예제 #11
0
    def generate_upload_tasks(self, job_id=None):
        """
        Generate upload tasks for the ingest job. This creates once task for each tile that has to be uploaded in the
        ingest queue

        Args:
            job_id: Job id of the ingest queue. If not included this takes the current ingest job

        Returns:
            None
        Raises:
            BossError : if there is no valid ingest job

        """

        if job_id is None and self.job is None:
            raise BossError(
                "Unable to generate upload tasks for the ingest service. Please specify a ingest job",
                ErrorCodes.UNABLE_TO_VALIDATE)
        elif job_id:
            # Using the job id to get the job
            try:
                ingest_job = IngestJob.objects.get(id=job_id)
            except IngestJob.DoesNotExist:
                raise BossError(
                    "Ingest job with id {} does not exist".format(job_id),
                    ErrorCodes.RESOURCE_NOT_FOUND)
        else:
            ingest_job = self.job

        # Generate upload tasks for the ingest job
        # Get the project information
        bosskey = ingest_job.collection + CONNECTER + ingest_job.experiment + CONNECTER + ingest_job.channel
        lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
        [col_id, exp_id, ch_id] = lookup_key.split('&')
        project_info = [col_id, exp_id, ch_id]

        # Batch messages and write to file
        base_file_name = 'tasks_' + lookup_key + '_' + str(ingest_job.id)
        self.file_index = 0

        # open file
        f = io.StringIO()
        header = {
            'job_id': ingest_job.id,
            'upload_queue_url': ingest_job.upload_queue,
            'ingest_queue_url': ingest_job.ingest_queue
        }
        f.write(json.dumps(header))
        f.write('\n')
        num_msg_per_file = 0

        for time_step in range(ingest_job.t_start, ingest_job.t_stop, 1):
            # For each time step, compute the chunks and tile keys

            for z in range(ingest_job.z_start, ingest_job.z_stop, 16):
                for y in range(ingest_job.y_start, ingest_job.y_stop,
                               ingest_job.tile_size_y):
                    for x in range(ingest_job.x_start, ingest_job.x_stop,
                                   ingest_job.tile_size_x):

                        # compute the chunk indices
                        chunk_x = int(x / ingest_job.tile_size_x)
                        chunk_y = int(y / ingest_job.tile_size_y)
                        chunk_z = int(z / 16)

                        # Compute the number of tiles in the chunk
                        if ingest_job.z_stop - z >= 16:
                            num_of_tiles = 16
                        else:
                            num_of_tiles = ingest_job.z_stop - z

                        # Generate the chunk key
                        chunk_key = (BossBackend(
                            self.config)).encode_chunk_key(
                                num_of_tiles, project_info,
                                ingest_job.resolution, chunk_x, chunk_y,
                                chunk_z, time_step)

                        self.num_of_chunks += 1

                        # get the tiles keys for this chunk
                        for tile in range(z, z + num_of_tiles):
                            # get the tile key
                            tile_key = (BossBackend(
                                self.config)).encode_tile_key(
                                    project_info, ingest_job.resolution,
                                    chunk_x, chunk_y, tile, time_step)
                            self.count_of_tiles += 1

                            # Generate the upload task msg
                            msg = chunk_key + ',' + tile_key + '\n'
                            f.write(msg)
                            num_msg_per_file += 1

                            # if there are 10 messages in the batch send it to the upload queue.
                            if num_msg_per_file == MAX_NUM_MSG_PER_FILE:
                                fname = base_file_name + '_' + str(
                                    self.file_index + 1) + '.txt'
                                self.upload_task_file(fname, f.getvalue())
                                self.file_index += 1
                                f.close()
                                # status = self.send_upload_message_batch(batch_msg)

                                fname = base_file_name + '_' + str(
                                    self.file_index + 1) + '.txt'
                                f = io.StringIO()
                                header = {
                                    'job_id': ingest_job.id,
                                    'upload_queue_url':
                                    ingest_job.upload_queue,
                                    'ingest_queue_url': ingest_job.ingest_queue
                                }
                                f.write(json.dumps(header))
                                f.write('\n')
                                num_msg_per_file = 0

        # Edge case: the last batch size maybe smaller than 10
        if num_msg_per_file != 0:
            fname = base_file_name + '_' + str(self.file_index + 1) + '.txt'
            self.upload_task_file(fname, f.getvalue())
            f.close()
            self.file_index += 1
            num_msg_per_file = 0

        # Update status
        self.job.tile_count = self.count_of_tiles
        self.job.save()
예제 #12
0
    def get(self,
            request,
            collection,
            experiment,
            dataset,
            orientation,
            resolution,
            x_args,
            y_args,
            z_args,
            t_args=None):
        """
        View to handle GET requests for a cuboid of data while providing all params

        :param request: DRF Request object
        :type request: rest_framework.request.Request
        :param collection: Unique Collection identifier, indicating which collection you want to access
        :param experiment: Experiment identifier, indicating which experiment you want to access
        :param dataset: Dataset identifier, indicating which channel or layer you want to access
        :param orientation: Image plane requested. Vaid options include xy,xz or yz
        :param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
        :param x_args: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
        :param y_args: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
        :param z_args: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
        :return:
        """
        # Process request and validate
        try:
            req = BossRequest(request)
        except BossError as err:
            return BossError.to_http()

        # Convert to Resource
        resource = spdb.project.BossResourceDjango(req)

        # Get bit depth
        try:
            self.bit_depth = resource.get_bit_depth()
        except ValueError:
            return BossHTTPError("Datatype does not match channel/layer",
                                 ErrorCodes.DATATYPE_DOES_NOT_MATCH)

        # Make sure cutout request is under 1GB UNCOMPRESSED
        total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span(
        ) * len(req.get_time()) * (self.bit_depth / 8)
        if total_bytes > settings.CUTOUT_MAX_SIZE:
            return BossHTTPError(
                "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.",
                ErrorCodes.REQUEST_TOO_LARGE)

        # Get interface to SPDB cache
        cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS,
                                         settings.STATEIO_CONFIG,
                                         settings.OBJECTIO_CONFIG)

        # Get the params to pull data out of the cache
        corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
        extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())

        # Do a cutout as specified
        data = cache.cutout(
            resource, corner, extent, req.get_resolution(),
            [req.get_time().start, req.get_time().stop])

        # Covert the cutout back to an image and return it
        if orientation == 'xy':
            img = data.xy_image()
        elif orientation == 'yz':
            img = data.yz_image()
        elif orientation == 'xz':
            img = data.xz_image()
        else:
            return BossHTTPError("Invalid orientation: {}".format(orientation),
                                 ErrorCodes.INVALID_CUTOUT_ARGS)

        return Response(img)
예제 #13
0
    def setup_ingest(self, creator, config_data):
        """
        Setup the ingest job. This is the primary method for the ingest manager.
        It creates the ingest job and queues required for the ingest. It also uploads the messages for the ingest

        Args:
            creator: The validated user from the request to create the ingest jon
            config_data : Config data to create the ingest job

        Returns:
            IngestJob : data model containing the ingest job

        Raises:
            BossError : For all exceptions that happen

        """
        # Validate config data and schema

        self.owner = creator
        try:
            valid_schema = self.validate_config_file(config_data)
            valid_prop = self.validate_properties()
            if valid_schema is True and valid_prop is True:
                # create the django model for the job
                self.job = self.create_ingest_job()

                # create the additional resources needed for the ingest
                # initialize the ndingest project for use with the library
                proj_class = BossIngestProj.load()
                self.nd_proj = proj_class(self.collection.name,
                                          self.experiment.name,
                                          self.channel.name, self.resolution,
                                          self.job.id)

                # Create the upload queue
                upload_queue = self.create_upload_queue()
                self.job.upload_queue = upload_queue.url

                # Create the ingest queue
                ingest_queue = self.create_ingest_queue()
                self.job.ingest_queue = ingest_queue.url

                # Call the step function to populate the queue.
                self.job.step_function_arn = self.populate_upload_queue()

                # Compute # of tiles in the job
                x_extent = self.job.x_stop - self.job.x_start
                y_extent = self.job.y_stop - self.job.y_start
                z_extent = self.job.z_stop - self.job.z_start
                t_extent = self.job.t_stop - self.job.t_start
                num_tiles_in_x = math.ceil(x_extent / self.job.tile_size_x)
                num_tiles_in_y = math.ceil(y_extent / self.job.tile_size_y)
                num_tiles_in_z = math.ceil(z_extent / self.job.tile_size_z)
                num_tiles_in_t = math.ceil(t_extent / self.job.tile_size_t)
                self.job.tile_count = num_tiles_in_x * num_tiles_in_y * num_tiles_in_z * num_tiles_in_t
                self.job.save()

                # tile_bucket = TileBucket(self.job.collection + '&' + self.job.experiment)
                # self.create_ingest_credentials(upload_queue, tile_bucket)

        except BossError as err:
            raise BossError(err.message, err.error_code)
        except Exception as e:
            raise BossError(
                "Unable to create the upload and ingest queue.{}".format(e),
                ErrorCodes.BOSS_SYSTEM_ERROR)
        return self.job
예제 #14
0
    def generate_upload_tasks(self, job_id=None):
        """

        Args:
            job_id:

        Returns:

        """

        if job_id is None and self.job is None:
            raise BossError(
                "Unable to generate upload tasks for the ingest service. Please specify a ingest job",
                ErrorCodes.UNABLE_TO_VALIDATE)
        elif job_id:
            # Using the job id to get the job
            try:
                ingest_job = IngestJob.objects.get(id=job_id)
            except IngestJob.DoesNotExist:
                raise BossError(
                    "Ingest job with id {} does not exist".format(job_id),
                    ErrorCodes.RESOURCE_NOT_FOUND)
        else:
            ingest_job = self.job

        # Generate upload tasks for the ingest job
        # Get the project information
        bosskey = ingest_job.collection + CONNECTER + ingest_job.experiment + CONNECTER + ingest_job.channel_layer
        lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
        [col_id, exp_id, ch_id] = lookup_key.split('&')
        project_info = [col_id, exp_id, ch_id]

        for time_step in range(ingest_job.t_start, ingest_job.t_stop, 1):
            # For each time step, compute the chunks and tile keys

            for z in range(ingest_job.z_start, ingest_job.z_stop, 16):
                for y in range(ingest_job.y_start, ingest_job.y_stop,
                               ingest_job.tile_size_y):
                    for x in range(ingest_job.x_start, ingest_job.x_stop,
                                   ingest_job.tile_size_x):

                        # compute the chunk indices
                        chunk_x = int(x / ingest_job.tile_size_x)
                        chunk_y = int(y / ingest_job.tile_size_y)
                        chunk_z = int(z / 16)

                        # Compute the number of tiles in the chunk
                        if ingest_job.z_stop - z >= 16:
                            num_of_tiles = 16
                        else:
                            num_of_tiles = ingest_job.z_stop - z

                        # Generate the chunk key
                        chunk_key = (BossBackend(
                            self.config)).encode_chunk_key(
                                num_of_tiles, project_info,
                                ingest_job.resolution, chunk_x, chunk_y,
                                chunk_z, time_step)
                        # get the tiles keys for this chunk
                        for tile in range(0, num_of_tiles):
                            # get the tile key
                            tile_key = (BossBackend(
                                self.config)).encode_tile_key(
                                    project_info, ingest_job.resolution,
                                    chunk_x, chunk_y, tile, time_step)

                            # Generate the upload task msg
                            msg = self.create_upload_task_message(
                                ingest_job.id, chunk_key, tile_key,
                                ingest_job.upload_queue,
                                ingest_job.ingest_queue)

                            # Upload the message
                            self.send_upload_task_message(msg)
예제 #15
0
파일: views.py 프로젝트: jhuapl-boss/boss
    def get(self, request, ingest_job_id=None):
        """
        Join a job with the specified job id or list all job ids if ingest_job_id is omitted
        Args:
            request: Django rest framework request object
            ingest_job_id: Ingest job id

        Returns:
            Ingest job
        """
        try:
            if ingest_job_id is None:
                # If the job ID is empty on a get, you are listing jobs
                return self.list_ingest_jobs(request)

            ingest_mgmr = IngestManager()
            ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)

            # Check permissions
            if not self.is_user_or_admin(request, ingest_job):
                return BossHTTPError(
                    "Only the creator or admin can join an ingest job",
                    ErrorCodes.INGEST_NOT_CREATOR)

            serializer = IngestJobListSerializer(ingest_job)

            # Start setting up output
            data = {'ingest_job': serializer.data}
            data['ingest_job']['tile_index_queue'] = None
            if ingest_job.ingest_type == IngestJob.TILE_INGEST:
                data['ingest_job'][
                    'tile_index_queue'] = ingest_mgmr.get_ingest_job_tile_index_queue(
                        ingest_job).url

            if ingest_job.status == IngestJob.DELETED:
                raise BossError(
                    "The job with id {} has been deleted".format(
                        ingest_job_id), ErrorCodes.INVALID_REQUEST)
            elif ingest_job.status == IngestJob.COMPLETE or ingest_job.status == IngestJob.FAILED:
                return Response(data, status=status.HTTP_200_OK)

            elif ingest_job.status == IngestJob.PREPARING:
                # check status of the step function
                session = bossutils.aws.get_session()
                if bossutils.aws.sfn_status(
                        session, ingest_job.step_function_arn) == 'SUCCEEDED':
                    # generate credentials
                    ingest_job.status = 1
                    ingest_job.save()
                    ingest_mgmr.generate_ingest_credentials(ingest_job)
                elif bossutils.aws.sfn_status(
                        session, ingest_job.step_function_arn) == 'FAILED':
                    # This indicates an error in step function
                    raise BossError(
                        "Error generating ingest job messages"
                        " Delete the ingest job with id {} and try again.".
                        format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)

            if ingest_job.status in [
                    IngestJob.UPLOADING, IngestJob.WAIT_ON_QUEUES,
                    IngestJob.COMPLETING
            ]:
                data['ingest_job']['status'] = ingest_job.status
                ingest_creds = IngestCredentials()
                data['credentials'] = ingest_creds.get_credentials(
                    ingest_job.id)
            else:
                data['credentials'] = None

            data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
            data['ingest_bucket_name'] = INGEST_BUCKET
            data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
            data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
            data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG

            # Strip out un-needed data from OBJECTIO_CONFIG to save space when
            # including in S3 metadata.
            data['OBJECTIO_CONFIG'].pop('index_deadletter_queue', None)
            data['OBJECTIO_CONFIG'].pop('index_cuboids_keys_queue', None)

            # Set resource
            data['resource'] = ingest_mgmr.get_resource_data(ingest_job_id)

            # ingest_lambda is no longer required by the backend.  The backend
            # gets the name of the ingest lambda from boss-manage/lib/names.py.
            # Keep providing it in case an older ingest client used (which
            # still expects it).
            data['ingest_lambda'] = 'deprecated'

            return Response(data, status=status.HTTP_200_OK)
        except BossError as err:
            return err.to_http()
        except Exception as err:
            return BossError("{}".format(err),
                             ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
예제 #16
0
    def try_start_completing(self, ingest_job):
        """
        Tries to start completion process.

        It is assumed that the ingest job status is currently WAIT_ON_QUEUES.

        If ingest_job status can be set to COMPLETING, then this process "wins"
        and starts the completion process.

        Args:
            ingest_job: Ingest job model

        Returns:
            (dict): { status: (job status str), wait_secs: (int) - # seconds client should wait }

        Raises:
            (BossError): If completion process cannot be started or is already
            in process.
        """
        completing_success = {
            'job_status': IngestJob.COMPLETING,
            'wait_secs': 0
        }

        if ingest_job.status == IngestJob.COMPLETING:
            return completing_success

        try:
            self.ensure_queues_empty(ingest_job)
        except BossError as be:
            # Ensure state goes back to UPLOADING if the upload queue isn't
            # empty.
            if be.message == UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG:
                ingest_job.status = IngestJob.UPLOADING
                ingest_job.save()
            raise

        if ingest_job.status != IngestJob.WAIT_ON_QUEUES:
            raise BossError(NOT_IN_WAIT_ON_QUEUES_STATE_ERR_MSG, ErrorCodes.BAD_REQUEST)

        wait_remaining = self.calculate_remaining_queue_wait(ingest_job)
        if wait_remaining > 0:
            return {
                'job_status': IngestJob.WAIT_ON_QUEUES,
                'wait_secs': wait_remaining
            }

        rows_updated = (IngestJob.objects
            .exclude(status=IngestJob.COMPLETING)
            .filter(id=ingest_job.id)
            .update(status=IngestJob.COMPLETING)
            )
        
        # If successfully set status to COMPLETING, kick off the completion
        # process.  Otherwise, completion already started.
        if rows_updated > 0:
            self._start_completion_activity(ingest_job)
            log = bossLogger()
            log.info(f"Started completion step function for job: {ingest_job.id}")


        return completing_success
예제 #17
0
파일: views.py 프로젝트: jhuapl-boss/boss
    def track_usage_data(self, ingest_config_data, request):
        """
        Set up usage tracking of this ingest.

        Args:
            ingest_config_data (dict): Ingest job config.

        Raises:
            (BossError): if user doesn't have permission for a large ingest.
        """

        # need to get bytes per pixel to caculate ingest in total bytes
        try:
            ingest_job = ingest_config_data['ingest_job']
            theCollection = Collection.objects.get(
                name=ingest_config_data['database']['collection'])
            theExperiment = Experiment.objects.get(
                name=ingest_config_data['database']['experiment'])
            theChannel = Channel.objects.get(
                name=ingest_config_data['database']['channel'])
            bytesPerPixel = int(theChannel.datatype.replace("uint", "")) / 8

            # Add metrics to CloudWatch
            extent = ingest_job['extent']
            database = ingest_config_data['database']

            # Check that only permitted users are creating extra large ingests
            try:
                group = Group.objects.get(name=INGEST_GRP)
                in_large_ingest_group = group.user_set.filter(
                    id=request.user.id).exists()
            except Group.DoesNotExist:
                # Just in case the group has not been created yet
                in_large_ingest_group = False
            if (not in_large_ingest_group) and \
            ((extent['x'][1] - extent['x'][0]) * \
                (extent['y'][1] - extent['y'][0]) * \
                (extent['z'][1] - extent['z'][0]) * \
                (extent['t'][1] - extent['t'][0]) > settings.INGEST_MAX_SIZE):
                raise BossError(
                    "Large ingests require special permission to create. Contact system administrator.",
                    ErrorCodes.INVALID_STATE)
            # Calculate the cost of the ingest in pixels
            costInPixels = ((extent['x'][1] - extent['x'][0]) *
                            (extent['y'][1] - extent['y'][0]) *
                            (extent['z'][1] - extent['z'][0]) *
                            (extent['t'][1] - extent['t'][0]))
            cost = costInPixels * bytesPerPixel
            BossThrottle().check('ingest', ThrottleMetric.METRIC_TYPE_INGRESS,
                                 request.user, cost,
                                 ThrottleMetric.METRIC_UNITS_BYTES)

            boss_config = bossutils.configuration.BossConfig()
            dimensions = [
                {
                    'Name': 'User',
                    'Value': request.user.username or "public"
                },
                {
                    'Name':
                    'Resource',
                    'Value':
                    '{}/{}/{}'.format(database['collection'],
                                      database['experiment'],
                                      database['channel'])
                },
                {
                    'Name': 'Stack',
                    'Value': boss_config['system']['fqdn']
                },
            ]

            session = bossutils.aws.get_session()
            client = session.client('cloudwatch')

            try:
                client.put_metric_data(Namespace="BOSS/Ingest",
                                       MetricData=[{
                                           'MetricName': 'InvokeCount',
                                           'Dimensions': dimensions,
                                           'Value': 1.0,
                                           'Unit': 'Count'
                                       }, {
                                           'MetricName': 'IngressCost',
                                           'Dimensions': dimensions,
                                           'Value': cost,
                                           'Unit': 'Bytes'
                                       }])
            except Exception as e:
                log = bossLogger()
                log.exception('Error during put_metric_data: {}'.format(e))
                log.exception('Allowing bossDB to continue after logging')

        except BossError as err:
            return err.to_http()
        except Exception as err:
            return BossError("{}".format(err),
                             ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
예제 #18
0
파일: views.py 프로젝트: ndrenkow/boss
    def get(self, request, ingest_job_id):
        """

        Args:
            job_id:

        Returns:

        """
        try:
            ingest_mgmr = IngestManager()
            ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
            serializer = IngestJobListSerializer(ingest_job)
            print(serializer.data)

            # Start setting up output
            data = {}
            data['ingest_job'] = serializer.data
            if ingest_job.status == 3 or ingest_job.status == 2:
                # Return the information for the deleted job/completed job
                return Response(data, status=status.HTTP_200_OK)
            elif ingest_job.status == 0:
                # check if all message are in the upload queue
                upload_queue = ingest_mgmr.get_ingest_job_upload_queue(
                    ingest_job)
                if int(upload_queue.queue.
                       attributes['ApproximateNumberOfMessages']) == int(
                           ingest_job.tile_count):
                    #generate credentials
                    ingest_job.status = 1
                    ingest_job.save()
                elif int(upload_queue.queue.
                         attributes['ApproximateNumberOfMessages']) > int(
                             ingest_job.tile_count):
                    # This indicates an error in the lambda
                    raise BossError(
                        "Error generating ingest job messages due to resources timing out ."
                        " Delete the ingest job with id {} and try again.".
                        format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)

            if ingest_job.status == 1:
                data['ingest_job']['status'] = 1
                ingest_creds = IngestCredentials()
                data['credentials'] = ingest_creds.get_credentials(
                    ingest_job.id)
            else:
                data['credentials'] = None

            data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
            data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
            data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
            data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG

            # add the lambda - Possibly remove this later
            config = bossutils.configuration.BossConfig()
            data['ingest_lambda'] = config["lambda"]["page_in_function"]

            # Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly
            collection = Collection.objects.get(
                name=data['ingest_job']["collection"])
            experiment = Experiment.objects.get(
                name=data['ingest_job']["experiment"], collection=collection)
            channel = Channel.objects.get(name=data['ingest_job']["channel"],
                                          experiment=experiment)

            resource = {}
            resource['boss_key'] = '{}&{}&{}'.format(
                data['ingest_job']["collection"],
                data['ingest_job']["experiment"],
                data['ingest_job']["channel"])
            resource['lookup_key'] = '{}&{}&{}'.format(collection.id,
                                                       experiment.id,
                                                       channel.id)
            resource['channel'] = {}
            resource['channel']['name'] = channel.name
            resource['channel']['description'] = ""
            resource['channel']['type'] = channel.type
            resource['channel']['datatype'] = channel.datatype
            resource['channel']['base_resolution'] = channel.base_resolution
            resource['channel']['sources'] = [
                x.name for x in channel.sources.all()
            ]
            resource['channel']['related'] = [
                x.name for x in channel.related.all()
            ]
            resource['channel'][
                'default_time_sample'] = channel.default_time_sample

            # Set resource
            data['resource'] = resource

            return Response(data, status=status.HTTP_200_OK)
        except BossError as err:
            return err.to_http()
        except Exception as err:
            return BossError("{}".format(err),
                             ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
예제 #19
0
파일: views.py 프로젝트: jhuapl-boss/boss
    def get(self, request, ingest_job_id):
        """
        Get the status of an ingest_job and number of messages in the upload queue
        Args:
            request: Django Rest framework object
            ingest_job_id: Ingest job id

        Returns: Status of the job

        """
        try:
            ingest_mgmr = IngestManager()
            ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)

            # Check if user is the ingest job creator or the sys admin
            if not self.is_user_or_admin(request, ingest_job):
                return BossHTTPError(
                    "Only the creator or admin can check the status of an ingest job",
                    ErrorCodes.INGEST_NOT_CREATOR)

            if ingest_job.status == IngestJob.DELETED:
                # Deleted Job
                raise BossError(
                    "The job with id {} has been deleted".format(
                        ingest_job_id), ErrorCodes.INVALID_REQUEST)
            else:
                if ingest_job.status == IngestJob.COMPLETE:
                    # Job is Complete so queues are gone
                    num_messages_in_queue = 0
                else:
                    try:
                        upload_queue = ingest_mgmr.get_ingest_job_upload_queue(
                            ingest_job)
                        num_messages_in_queue = int(
                            upload_queue.queue.
                            attributes['ApproximateNumberOfMessages'])
                        if num_messages_in_queue < ingest_job.tile_count:
                            for n in range(9):
                                num_messages_in_queue += int(
                                    upload_queue.queue.
                                    attributes['ApproximateNumberOfMessages'])
                            num_messages_in_queue /= 10
                    except Exception:
                        if ingest_job.status != IngestJob.COMPLETING:
                            raise

                        # Probably threw because queues were deleted while
                        # completing ingest.
                        num_messages_in_queue = 0

                data = {
                    "id": ingest_job.id,
                    "status": ingest_job.status,
                    "total_message_count": ingest_job.tile_count,
                    "current_message_count": int(num_messages_in_queue)
                }

            return Response(data, status=status.HTTP_200_OK)
        except BossError as err:
            return err.to_http()
        except Exception as err:
            return BossError("{}".format(err),
                             ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
예제 #20
0
파일: views.py 프로젝트: jhuapl-boss/boss
    def post(self, request, ingest_job_id):
        """
        Signal an ingest job is complete and should be cleaned up by POSTing to this view

        Args:
            request: Django Rest framework Request object
            ingest_job_id: Ingest job id

        Returns:


        """
        try:
            blog = bossLogger()
            ingest_mgmr = IngestManager()
            ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)

            # Check if user is the ingest job creator or the sys admin
            if not self.is_user_or_admin(request, ingest_job):
                return BossHTTPError(
                    "Only the creator or admin can complete an ingest job",
                    ErrorCodes.INGEST_NOT_CREATOR)

            if ingest_job.status == IngestJob.PREPARING:
                # If status is Preparing. Deny
                return BossHTTPError(
                    "You cannot complete a job that is still preparing. You must cancel instead.",
                    ErrorCodes.BAD_REQUEST)
            elif ingest_job.status == IngestJob.UPLOADING:
                try:
                    data = ingest_mgmr.try_enter_wait_on_queue_state(
                        ingest_job)
                    return Response(data=data, status=status.HTTP_202_ACCEPTED)
                except BossError as be:
                    if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG or
                            be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG):
                        # If there are messages in the tile error queue, this
                        # will have to be handled manually.  Non-empty ingest
                        # or tile index queues should resolve on their own.
                        return Response(data={
                            'wait_secs': WAIT_FOR_QUEUES_SECS,
                            'info': 'Internal queues not empty yet'
                        },
                                        status=status.HTTP_400_BAD_REQUEST)
                    raise
            elif ingest_job.status == IngestJob.WAIT_ON_QUEUES:
                pass
                # Continue below.
            elif ingest_job.status == IngestJob.COMPLETE:
                # If status is already Complete, just return another 204
                return Response(data={'job_status': ingest_job.status},
                                status=status.HTTP_204_NO_CONTENT)
            elif ingest_job.status == IngestJob.DELETED:
                # Job had already been cancelled
                return BossHTTPError("Ingest job has already been cancelled.",
                                     ErrorCodes.BAD_REQUEST)
            elif ingest_job.status == IngestJob.FAILED:
                # Job had failed
                return BossHTTPError(
                    "Ingest job has failed during creation. You must Cancel instead.",
                    ErrorCodes.BAD_REQUEST)
            elif ingest_job.status == IngestJob.COMPLETING:
                return Response(data={'job_status': ingest_job.status},
                                status=status.HTTP_202_ACCEPTED)

            # Check if user is the ingest job creator or the sys admin
            if not self.is_user_or_admin(request, ingest_job):
                return BossHTTPError(
                    "Only the creator or admin can complete an ingest job",
                    ErrorCodes.INGEST_NOT_CREATOR)

            # Try to start completing.
            try:
                data = ingest_mgmr.try_start_completing(ingest_job)
                if data['job_status'] == IngestJob.WAIT_ON_QUEUES:
                    # Refuse complete requests until wait period expires.
                    return Response(data=data,
                                    status=status.HTTP_400_BAD_REQUEST)
            except BossError as be:
                if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG
                        or be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG
                        or be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG):
                    return Response(data={
                        'wait_secs': WAIT_FOR_QUEUES_SECS,
                        'info': 'Internal queues not empty yet'
                    },
                                    status=status.HTTP_400_BAD_REQUEST)
                raise

            blog.info("Completion process started for ingest Job {}".format(
                ingest_job_id))
            return Response(data=data, status=status.HTTP_202_ACCEPTED)

            # TODO SH This is a quick fix to make sure the ingest-client does not run close option.
            #      the clean up code commented out below, because it is not working correctly.
            # return Response(status=status.HTTP_204_NO_CONTENT)

            # if ingest_job.ingest_type == IngestJob.TILE_INGEST:
            #     # Check if any messages remain in the ingest queue
            #     ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(ingest_job)
            #     num_messages_in_queue = int(ingest_queue.queue.attributes['ApproximateNumberOfMessages'])
            #
            #     # Kick off extra lambdas just in case
            #     if num_messages_in_queue:
            #         blog.info("{} messages remaining in Ingest Queue".format(num_messages_in_queue))
            #         ingest_mgmr.invoke_ingest_lambda(ingest_job, num_messages_in_queue)
            #
            #         # Give lambda a few seconds to fire things off
            #         time.sleep(30)
            #
            #     ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
            #
            # elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
            #     ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
            #
            # # ToDo: call cleanup method for volumetric ingests.  Don't want
            # # to cleanup until after testing with real data.
            # #ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
            #
            # blog.info("Complete successful")
            # return Response(status=status.HTTP_204_NO_CONTENT)
        except BossError as err:
            return err.to_http()
        except Exception as err:
            blog.error('Caught general exception: {}'.format(err))
            return BossError("{}".format(err),
                             ErrorCodes.BOSS_SYSTEM_ERROR).to_http()