コード例 #1
0
    def test_not_creator_admin(self):
        """Method to test only creators or admins can interact with ingest jobs"""
        config_data = self.setup_helper.get_ingest_config_data_dict()
        config_data = json.loads(json.dumps(config_data))

        # # Post the data
        url = '/' + version + '/ingest/'
        response = self.client.post(url, data=config_data, format='json')
        self.assertEqual(201, response.status_code)

        # Check if the queue's exist
        ingest_job = response.json()
        proj_class = BossIngestProj.load()
        nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
                             0, ingest_job['id'])
        self.nd_proj = nd_proj
        upload_queue = UploadQueue(nd_proj, endpoint_url=None)
        self.assertIsNotNone(upload_queue)
        ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
        self.assertIsNotNone(ingest_queue)

        # Log in as the admin and create a job
        self.client.force_login(self.superuser)

        # Test joining the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.get(url)
        self.assertEqual(200, response.status_code)
        self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
        self.assertIn("credentials", response.json())

        # # Delete the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.delete(url)
        self.assertEqual(204, response.status_code)
コード例 #2
0
    def test_generate_upload_tasks(self):
        """"""
        try:
            ingest_mgmr = IngestManager()
            ingest_job = ingest_mgmr.setup_ingest(self.user.id, self.example_config_data)
            ingest_mgmr.generate_upload_tasks(ingest_job.id)
            assert (ingest_job.collection == 'my_col_1')
            assert (ingest_job.experiment == 'my_exp_1')
            assert (ingest_job.channel == 'my_ch_1')

            # Pull the messages off the queue
            proj_class = BossIngestProj.load()
            nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
                             ingest_job.resolution, ingest_job.id)
            queue = UploadQueue(nd_proj, endpoint_url=None)

            tmp = queue.receiveMessage(number_of_messages=10)
            # receive message from the queue
            for message_id, receipt_handle, message_body in tmp:
                assert(message_body['job_id'] == ingest_job.id)

                # delete message from the queue
                response = queue.deleteMessage(message_id, receipt_handle)
                assert ('Successful' in response)
            ingest_mgmr.remove_ingest_credentials(ingest_job.id)

        except:
            raise
        finally:
            ingest_mgmr.delete_upload_queue()
            ingest_mgmr.delete_ingest_queue()
コード例 #3
0
    def test_setup_ingest(self):
        """Method to test the setup_ingest method"""
        try:
            ingest_mgmr = IngestManager()
            ingest_job = ingest_mgmr.setup_ingest(self.user.id,
                                                  self.example_config_data)
            assert (ingest_job is not None)

            # Check if the queue's exist
            proj_class = BossIngestProj.load()
            nd_proj = proj_class(ingest_job.collection, ingest_job.experiment,
                                 ingest_job.channel, ingest_job.resolution,
                                 ingest_job.id)
            ingest_mgmr.nd_proj = nd_proj
            upload_queue = UploadQueue(nd_proj, endpoint_url=None)
            assert (upload_queue is not None)
            ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
            assert (ingest_queue is not None)
            ingest_mgmr.remove_ingest_credentials(ingest_job.id)

        except:
            raise
        finally:
            ingest_mgmr.delete_upload_queue()
            ingest_mgmr.delete_ingest_queue()
コード例 #4
0
    def test_not_creator_admin(self):
        """Method to test only creators or admins can interact with ingest jobs"""
        config_data = self.setup_helper.get_ingest_config_data_dict()
        config_data = json.loads(json.dumps(config_data))

        # # Post the data
        url = '/' + version + '/ingest/'
        response = self.client.post(url, data=config_data, format='json')
        assert (response.status_code == 201)

        # Check if the queue's exist
        ingest_job = response.json()
        proj_class = BossIngestProj.load()
        nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
                             0, ingest_job['id'])
        self.nd_proj = nd_proj
        upload_queue = UploadQueue(nd_proj, endpoint_url=None)
        assert (upload_queue is not None)
        ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
        assert (ingest_queue is not None)

        # Log in as the admin and create a job
        self.client.force_login(self.superuser)

        # Test joining the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.get(url)
        assert (response.status_code == 200)
        assert (response.json()['ingest_job']['id'] == ingest_job['id'])
        assert("credentials" in response.json())

        # # Delete the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.delete(url)
        assert (response.status_code == 204)
コード例 #5
0
    def test_post_new_ingest_job(self):
        """ Test view to create a new ingest job """

        config_data = self.setup_helper.get_ingest_config_data_dict()
        config_data = json.loads(json.dumps(config_data))

        # # Post the data
        url = '/' + version + '/ingest/'
        response = self.client.post(url, data=config_data, format='json')
        assert (response.status_code == 201)

        # Check if the queue's exist
        ingest_job = response.json()
        proj_class = BossIngestProj.load()
        nd_proj = proj_class(ingest_job['collection'],
                             ingest_job['experiment'],
                             ingest_job['channel_layer'], 0, ingest_job['id'])
        self.nd_proj = nd_proj
        upload_queue = UploadQueue(nd_proj, endpoint_url=None)
        assert (upload_queue is not None)
        ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
        assert (ingest_queue is not None)

        # # Delete the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.delete(url)
        assert (response.status_code == 204)
コード例 #6
0
    def setup_ingest(self, creator, config_data):
        """
        Setup the ingest job. This is the primary method for the ingest manager.
        It creates the ingest job and queues required for the ingest. It also uploads the messages for the ingest

        Args:
            creator: The validated user from the request to create the ingest jon
            config_data : Config data to create the ingest job

        Returns:
            IngestJob : data model containing the ingest job

        Raises:
            BossError : For all exceptions that happen

        """
        # Validate config data and schema

        self.owner = creator
        try:
            valid_schema = self.validate_config_file(config_data)
            valid_prop = self.validate_properties()
            if valid_schema is True and valid_prop is True:
                # create the django model for the job
                self.job = self.create_ingest_job()

                # create the additional resources needed for the ingest
                # initialize the ndingest project for use with the library
                proj_class = BossIngestProj.load()
                self.nd_proj = proj_class(self.collection.name,
                                          self.experiment.name,
                                          self.channel.name, self.resolution,
                                          self.job.id)

                # Create the upload queue
                upload_queue = self.create_upload_queue()
                self.job.upload_queue = upload_queue.url

                # Create the ingest queue
                ingest_queue = self.create_ingest_queue()
                self.job.ingest_queue = ingest_queue.url

                self.generate_upload_tasks()
                tile_bucket = TileBucket(self.job.collection + '&' +
                                         self.job.experiment)

                self.create_ingest_credentials(upload_queue, tile_bucket)

            # TODO create channel if needed

        except BossError as err:
            raise BossError(err.message, err.error_code)
        except Exception as e:
            raise BossError(
                "Unable to create the upload and ingest queue.{}".format(e),
                ErrorCodes.BOSS_SYSTEM_ERROR)
        return self.job
コード例 #7
0
    def setup_ingest(self, creator, config_data):
        """

        Args:


        Returns:

        """
        # Validate config data and schema

        self.owner = creator
        try:
            valid_schema = self.validate_config_file(config_data)
            valid_prop = self.validate_properties()
            if valid_schema is True and valid_prop is True:
                # create the django model for the job
                self.job = self.create_ingest_job()

                # create the additional resources needed for the ingest
                # initialize the ndingest project for use with the library
                proj_class = BossIngestProj.load()
                self.nd_proj = proj_class(self.collection.name,
                                          self.experiment.name,
                                          self.channel_layer.name,
                                          self.resolution, self.job.id)

                # Create the upload queue
                upload_queue = self.create_upload_queue()
                self.job.upload_queue = upload_queue.url

                # Create the ingest queue
                ingest_queue = self.create_ingest_queue()
                self.job.ingest_queue = ingest_queue.url

                self.generate_upload_tasks()
                tile_bucket = TileBucket(self.job.collection + '&' +
                                         self.job.experiment)

                self.create_ingest_credentials(upload_queue, tile_bucket)

                # Update status
                self.job.status = 1
                self.job.save()

            # TODO create channel if needed

        except BossError as err:
            raise BossError(err.message, err.error_code)
        except Exception as e:
            raise BossError(
                "Unable to create the upload and ingest queue.{}".format(e),
                ErrorCodes.BOSS_SYSTEM_ERROR)
        return self.job
コード例 #8
0
    def test_complete_ingest_job(self):
        """ Test view to create a new ingest job """
        config_data = self.setup_helper.get_ingest_config_data_dict()
        config_data = json.loads(json.dumps(config_data))

        # # Post the data
        url = '/' + version + '/ingest/'
        response = self.client.post(url, data=config_data, format='json')
        self.assertEqual(response.status_code, 201)

        # Check if the queues exist
        ingest_job = response.json()
        proj_class = BossIngestProj.load()
        nd_proj = proj_class(ingest_job['collection'],
                             ingest_job['experiment'], ingest_job['channel'],
                             0, ingest_job['id'])
        self.nd_proj = nd_proj
        upload_queue = UploadQueue(nd_proj, endpoint_url=None)
        self.assertIsNotNone(upload_queue)
        ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
        self.assertIsNotNone(ingest_queue)

        # Test joining the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.get(url)
        self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
        self.assertIn("credentials", response.json())

        # Complete the job
        url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
        response = self.client.post(url)

        # Can't complete until it is done
        self.assertEqual(400, response.status_code)

        # Wait for job to complete
        print('trying to join job')
        for cnt in range(0, 30):
            # Try joining to kick the status
            url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
            self.client.get(url)

            url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
            response = self.client.get(url)
            if response.json()["status"] == IngestJob.UPLOADING:
                break

            time.sleep(10)

        print('completing')
        # Complete the job
        url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
        response = self.client.post(url)
        self.assertEqual(204, response.status_code)
コード例 #9
0
    def test_complete_ingest_job(self):
        """ Test view to create a new ingest job """
        config_data = self.setup_helper.get_ingest_config_data_dict()
        config_data = json.loads(json.dumps(config_data))

        # # Post the data
        url = '/' + version + '/ingest/'
        response = self.client.post(url, data=config_data, format='json')
        self.assertEqual(response.status_code, 201)

        # Check if the queues exist
        ingest_job = response.json()
        proj_class = BossIngestProj.load()
        nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
                             0, ingest_job['id'])
        self.nd_proj = nd_proj
        upload_queue = UploadQueue(nd_proj, endpoint_url=None)
        self.assertIsNotNone(upload_queue)
        ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
        self.assertIsNotNone(ingest_queue)

        # Test joining the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.get(url)
        self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
        self.assertIn("credentials", response.json())

        # Complete the job
        url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
        response = self.client.post(url)

        # Can't complete until it is done
        self.assertEqual(400, response.status_code)

        # Wait for job to complete
        print('trying to join job')
        for cnt in range(0, 30):
            # Try joining to kick the status
            url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
            self.client.get(url)

            url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
            response = self.client.get(url)
            if response.json()["status"] == IngestJob.UPLOADING:
                break

            time.sleep(10)

        print('completing')
        # Complete the job
        url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
        response = self.client.post(url)
        self.assertEqual(204, response.status_code)
コード例 #10
0
    def get_ingest_job_tile_error_queue(self, ingest_job):
        """
        Return the tile index queue for an ingest job
        Args:
            ingest_job: Ingest job model

        Returns:
            ndingest.TileIndexQueue
        """
        proj_class = BossIngestProj.load()
        self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
                                  ingest_job.resolution, ingest_job.id)
        queue = TileErrorQueue(self.nd_proj, endpoint_url=None)
        return queue
コード例 #11
0
ファイル: ingest_manager.py プロジェクト: jhuapl-boss/boss
    def get_ingest_job_ingest_queue(self, ingest_job):
        """
        Return the ingest queue for an ingest job
        Args:
            ingest_job: Ingest job model

        Returns:
            Ndingest.ingestqueue
        """
        proj_class = BossIngestProj.load()
        self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
                                  ingest_job.resolution, ingest_job.id)
        queue = IngestQueue(self.nd_proj, endpoint_url=None)
        return queue
コード例 #12
0
ファイル: ingest_manager.py プロジェクト: jhuapl-boss/boss
    def get_ingest_job_tile_error_queue(self, ingest_job):
        """
        Return the tile index queue for an ingest job
        Args:
            ingest_job: Ingest job model

        Returns:
            ndingest.TileIndexQueue
        """
        proj_class = BossIngestProj.load()
        self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
                                  ingest_job.resolution, ingest_job.id)
        queue = TileErrorQueue(self.nd_proj, endpoint_url=None)
        return queue
コード例 #13
0
    def get_ingest_job_upload_queue(self, ingest_job):
        """
        Return the upload queue for an ingest job
        Args:
            ingest_job: Ingest job model

        Returns:
            Ndingest.uploadqueue
        """
        proj_class = BossIngestProj.load()
        self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
                                  ingest_job.resolution, ingest_job.id)
        queue = UploadQueue(self.nd_proj, endpoint_url=None)
        return queue
コード例 #14
0
    def cleanup_ingest_job(self, ingest_job, job_status):
        """
        Delete or complete an ingest job with a specific id. Note this deletes the queues, credentials and all the remaining tiles
        in the tile bucket for this job id. It does not delete the ingest job datamodel but changes its state.
        Args:
            ingest_job: Ingest job to cleanup
            job_status(int): Status to update to

        Returns:
            (int): ingest job id for the job that was successfully deleted

        Raises:
            BossError : If the the job id is not valid or any exception happens in deletion process

        """
        try:
            # cleanup ingest job
            proj_class = BossIngestProj.load()
            self.nd_proj = proj_class(ingest_job.collection,
                                      ingest_job.experiment,
                                      ingest_job.channel,
                                      ingest_job.resolution, ingest_job.id)

            # delete the ingest and upload_queue
            self.delete_upload_queue()
            if ingest_job.ingest_type != IngestJob.VOLUMETRIC_INGEST:
                self.delete_ingest_queue()

            # delete any pending entries in the tile index database and tile bucket
            # Commented out due to removal of tile index's GSI.
            # self.delete_tiles(ingest_job)

            ingest_job.status = job_status
            ingest_job.ingest_queue = None
            ingest_job.upload_queue = None
            ingest_job.end_date = timezone.now()
            ingest_job.save()

            # Remove ingest credentials for a job
            self.remove_ingest_credentials(ingest_job.id)

        except Exception as e:
            raise BossError("Unable to cleanup the upload queue.{}".format(e),
                            ErrorCodes.BOSS_SYSTEM_ERROR)
        except IngestJob.DoesNotExist:
            raise BossError(
                "Ingest job with id {} does not exist".format(ingest_job.id),
                ErrorCodes.OBJECT_NOT_FOUND)
        return ingest_job.id
コード例 #15
0
    def delete_ingest_job(self, ingest_job_id):
        """
        Delete an ingest job with a specific id. Note this deletes the queues, credentials and all the remaining tiles
        in the tile bucket for this job id. It does not delete the ingest job datamodel but marks it as deleted.
        Args:
            ingest_job_id: Ingest job id to delete

        Returns:
            Int : ingest job id for the job that was successfully deleted

        Raises:
            BossError : If the the job id is not valid or any exception happens in deletion process

        """
        try:

            # delete ingest job
            ingest_job = IngestJob.objects.get(id=ingest_job_id)
            proj_class = BossIngestProj.load()
            self.nd_proj = proj_class(ingest_job.collection,
                                      ingest_job.experiment,
                                      ingest_job.channel,
                                      ingest_job.resolution, ingest_job.id)

            # delete the ingest and upload_queue
            self.delete_upload_queue()
            self.delete_ingest_queue()

            # delete any pending entries in the tile index database and tile bucket
            self.delete_tiles(ingest_job)

            ingest_job.status = 3
            ingest_job.ingest_queue = None
            ingest_job.upload_queue = None
            ingest_job.save()

            # Remove ingest credentials for a job
            self.remove_ingest_credentials(ingest_job_id)

        except Exception as e:
            raise BossError("Unable to delete the upload queue.{}".format(e),
                            ErrorCodes.BOSS_SYSTEM_ERROR)
        except IngestJob.DoesNotExist:
            raise BossError(
                "Ingest job with id {} does not exist".format(ingest_job_id),
                ErrorCodes.OBJECT_NOT_FOUND)
        return ingest_job_id
コード例 #16
0
ファイル: ingest_manager.py プロジェクト: jhuapl-boss/boss
    def cleanup_ingest_job(self, ingest_job, job_status):
        """
        Delete or complete an ingest job with a specific id. Note this deletes the queues, credentials and all the remaining tiles
        in the tile bucket for this job id. It does not delete the ingest job datamodel but changes its state.
        Args:
            ingest_job: Ingest job to cleanup
            job_status(int): Status to update to

        Returns:
            (int): ingest job id for the job that was successfully deleted

        Raises:
            BossError : If the the job id is not valid or any exception happens in deletion process

        """
        try:
            # cleanup ingest job
            proj_class = BossIngestProj.load()
            self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
                                      ingest_job.resolution, ingest_job.id)

            # delete the queues
            self.delete_upload_queue()
            if ingest_job.ingest_type != IngestJob.VOLUMETRIC_INGEST:
                self.delete_ingest_queue()
                self.delete_tile_index_queue()
                self.delete_tile_error_queue()

            # delete any pending entries in the tile index database and tile bucket
            # Commented out due to removal of tile index's GSI.
            # self.delete_tiles(ingest_job)

            ingest_job.status = job_status
            ingest_job.ingest_queue = None
            ingest_job.upload_queue = None
            ingest_job.end_date = timezone.now()
            ingest_job.save()

            # Remove ingest credentials for a job
            self.remove_ingest_credentials(ingest_job.id)

        except Exception as e:
            raise BossError("Unable to cleanup the upload queue.{}".format(e), ErrorCodes.BOSS_SYSTEM_ERROR)
        except IngestJob.DoesNotExist:
            raise BossError("Ingest job with id {} does not exist".format(ingest_job.id), ErrorCodes.OBJECT_NOT_FOUND)
        return ingest_job.id
コード例 #17
0
    def test_post_new_volumetric_ingest_job(self):
        """ Test view to create a new volumetric_ingest job """
        config_data = self.setup_helper.get_ingest_config_data_dict_volumetric(
        )
        config_data = json.loads(json.dumps(config_data))

        # # Post the data
        url = '/' + version + '/ingest/'
        response = self.client.post(url, data=config_data, format='json')
        self.assertEqual(201, response.status_code)

        # Check if the queue's exist
        ingest_job = response.json()
        proj_class = BossIngestProj.load()
        nd_proj = proj_class(ingest_job['collection'],
                             ingest_job['experiment'], ingest_job['channel'],
                             0, ingest_job['id'])
        self.nd_proj = nd_proj
        upload_queue = UploadQueue(nd_proj, endpoint_url=None)
        self.assertIsNotNone(upload_queue)

        # There shouldn't be an ingest queue for a volumetric ingest
        with self.assertRaises(ClientError):
            IngestQueue(nd_proj, endpoint_url=None)

        # Test joining the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.get(url)
        self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
        self.assertIn("credentials", response.json())

        # # Delete the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.delete(url)
        self.assertEqual(204, response.status_code)

        # Verify Queues are removed
        with self.assertRaises(ClientError):
            UploadQueue(nd_proj, endpoint_url=None)

        # Verify the job is deleted
        url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
        response = self.client.get(url)
        self.assertEqual(response.status_code, 404)
コード例 #18
0
    def test_post_new_ingest_job(self):
        """ Test view to create a new ingest job """
        config_data = self.setup_helper.get_ingest_config_data_dict()
        config_data = json.loads(json.dumps(config_data))

        # # Post the data
        url = '/' + version + '/ingest/'
        response = self.client.post(url, data=config_data, format='json')
        self.assertEqual(201, response.status_code)

        # Check if the queue's exist
        ingest_job = response.json()
        proj_class = BossIngestProj.load()
        nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
                             0, ingest_job['id'])
        self.nd_proj = nd_proj
        upload_queue = UploadQueue(nd_proj, endpoint_url=None)
        self.assertIsNotNone(upload_queue)
        ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
        self.assertIsNotNone(ingest_queue)

        # Test joining the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.get(url)
        self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
        self.assertIn("credentials", response.json())

        # # Delete the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.delete(url)
        self.assertEqual(204, response.status_code)

        # Verify Queues are removed
        with self.assertRaises(ClientError):
            UploadQueue(nd_proj, endpoint_url=None)
        with self.assertRaises(ClientError):
            IngestQueue(nd_proj, endpoint_url=None)

        # Verify the job is deleted
        url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
        response = self.client.get(url)
        self.assertEqual(response.status_code, 404)
コード例 #19
0
    def delete_ingest_job(self, ingest_job_id):
        """

        Args:
            ingest_job_id:

        Returns:

        """
        try:

            # delete ingest job
            ingest_job = IngestJob.objects.get(id=ingest_job_id)
            proj_class = BossIngestProj.load()
            self.nd_proj = proj_class(ingest_job.collection,
                                      ingest_job.experiment,
                                      ingest_job.channel_layer,
                                      ingest_job.resolution, ingest_job.id)

            # delete the ingest and upload_queue
            self.delete_upload_queue()
            self.delete_ingest_queue()

            # delete any pending entries in the tile index database and tile bucket
            self.delete_tiles(ingest_job)

            ingest_job.status = 3
            ingest_job.save()

            # Remove ingest credentials for a job
            self.remove_ingest_credentials(ingest_job_id)

        except Exception as e:
            raise BossError("Unable to delete the upload queue.{}".format(e),
                            ErrorCodes.BOSS_SYSTEM_ERROR)
        except IngestJob.DoesNotExist:
            raise BossError(
                "Ingest job with id {} does not exist".format(ingest_job_id),
                ErrorCodes.OBJECT_NOT_FOUND)
        return ingest_job_id
コード例 #20
0
    def __init__(self, status, db_host, job):
        """
        Args:
            status (str): Mark ingest job as 'deleted' or 'complete'.
            db_host (str): Host of MySQL database.
            job (dict):
                collection (int): Collection id.
                experiment (int): Experiment id.
                channel (int): Channel id.
                task_id (int): The ingest job's id.
                resolution (int): Resolution of chunk.
                ingest_type (int): Tile (0) or volumetric ingest (1).
        """
        self._status_map = {
            COMPLETE_STATUS: COMPLETE_DB,
            DELETED_STATUS: DELETED_DB
        }
        self.db_host = db_host
        self.job = job

        # Validate job parameter.
        for field in IngestCleaner.JOB_FIELDS:
            if field not in job:
                raise KeyError('Job must have {}'.format(field))

        # Validate status parameter.
        self.status = status
        if status.lower() not in IngestCleaner.STATUS_VALUES:
            raise BadStatusError('{} is not a valid status.'.format(status))

        proj_class = BossIngestProj.load()

        # coll/exp/chan are specified as strings, but not necessarily to give
        # actual names when using ndingest's delete functionality.
        self.nd_proj = proj_class(job['collection'], job['experiment'],
                                  job['channel'], job['resolution'],
                                  job['task_id'])
コード例 #21
0
ファイル: ingest_manager.py プロジェクト: jhuapl-boss/boss
    def setup_ingest(self, creator, config_data):
        """
        Setup the ingest job. This is the primary method for the ingest manager.
        It creates the ingest job and queues required for the ingest. It also uploads the messages for the ingest

        Args:
            creator: The validated user from the request to create the ingest jon
            config_data : Config data to create the ingest job

        Returns:
            IngestJob : data model containing the ingest job

        Raises:
            BossError : For all exceptions that happen

        """
        # Validate config data and schema

        self.owner = creator
        try:
            valid_schema = self.validate_config_file(config_data)
            valid_prop = self.validate_properties()
            if valid_schema is True and valid_prop is True:
                # create the django model for the job
                self.job = self.create_ingest_job()

                # create the additional resources needed for the ingest
                # initialize the ndingest project for use with the library
                proj_class = BossIngestProj.load()
                self.nd_proj = proj_class(self.collection.name, self.experiment.name, self.channel.name,
                                          self.resolution, self.job.id)

                # Create the upload queue
                upload_queue = self.create_upload_queue()
                self.job.upload_queue = upload_queue.url

                # Create the ingest queue
                if self.job.ingest_type == IngestJob.TILE_INGEST:
                    ingest_queue = self.create_ingest_queue()
                    self.job.ingest_queue = ingest_queue.url
                    tile_index_queue = self.create_tile_index_queue()
                    self.add_trigger_tile_uploaded_lambda_from_queue(tile_index_queue.arn)
                    self.create_tile_error_queue()
                elif self.job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
                    # Will the management console be ok with ingest_queue being null?
                    pass

                # Call the step function to populate the queue.
                self.job.step_function_arn = self.populate_upload_queue(self.job)

                # Compute # of tiles or chunks in the job
                x_extent = self.job.x_stop - self.job.x_start
                y_extent = self.job.y_stop - self.job.y_start
                z_extent = self.job.z_stop - self.job.z_start
                t_extent = self.job.t_stop - self.job.t_start
                num_tiles_in_x = math.ceil(x_extent/self.job.tile_size_x)
                num_tiles_in_y = math.ceil(y_extent/self.job.tile_size_y)
                num_tiles_in_z = math.ceil(z_extent/self.job.tile_size_z)
                num_tiles_in_t = math.ceil(t_extent / self.job.tile_size_t)
                self.job.tile_count = num_tiles_in_x * num_tiles_in_y * num_tiles_in_z * num_tiles_in_t
                self.job.save()

        except BossError as err:
            raise BossError(err.message, err.error_code)
        except Exception as e:
            raise BossError("Unable to create the upload and ingest queue.{}".format(e),
                            ErrorCodes.BOSS_SYSTEM_ERROR)
        return self.job
コード例 #22
0
    def test_complete_ingest_job(self):
        """ Test view to create a new ingest job """
        config_data = self.setup_helper.get_ingest_config_data_dict()
        config_data = json.loads(json.dumps(config_data))

        # # Post the data
        url = '/' + version + '/ingest/'
        response = self.client.post(url, data=config_data, format='json')
        assert (response.status_code == 201)

        # Check if the queue's exist
        ingest_job = response.json()
        proj_class = BossIngestProj.load()
        nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
                             0, ingest_job['id'])
        self.nd_proj = nd_proj
        upload_queue = UploadQueue(nd_proj, endpoint_url=None)
        assert (upload_queue is not None)
        ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
        assert (ingest_queue is not None)

        # Test joining the job
        url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
        response = self.client.get(url)
        assert(response.json()['ingest_job']['id'] == ingest_job['id'])
        assert("credentials" in response.json())

        # Complete the job
        url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
        response = self.client.post(url)

        # Can't complete until it is done
        assert(response.status_code == 400)

        # Wait for job to complete
        for cnt in range(0, 30):
            # Try joining to kick the status
            url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
            self.client.get(url)

            url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
            response = self.client.get(url)
            if response.json()["status"] == 1:
                break

            time.sleep(10)

        # Complete the job
        url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
        response = self.client.post(url)
        assert(response.status_code == 204)

        # Verify Queues are removed
        with self.assertRaises(ClientError):
            UploadQueue(nd_proj, endpoint_url=None)
        with self.assertRaises(ClientError):
            IngestQueue(nd_proj, endpoint_url=None)

        # Verify status has changed
        url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
        response = self.client.get(url)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.json()["status"], 2)
コード例 #23
0
    def setup_ingest(self, creator, config_data):
        """
        Setup the ingest job. This is the primary method for the ingest manager.
        It creates the ingest job and queues required for the ingest. It also uploads the messages for the ingest

        Args:
            creator: The validated user from the request to create the ingest jon
            config_data : Config data to create the ingest job

        Returns:
            IngestJob : data model containing the ingest job

        Raises:
            BossError : For all exceptions that happen

        """
        # Validate config data and schema

        self.owner = creator
        try:
            valid_schema = self.validate_config_file(config_data)
            valid_prop = self.validate_properties()
            if valid_schema is True and valid_prop is True:
                # create the django model for the job
                self.job = self.create_ingest_job()

                # create the additional resources needed for the ingest
                # initialize the ndingest project for use with the library
                proj_class = BossIngestProj.load()
                self.nd_proj = proj_class(self.collection.name,
                                          self.experiment.name,
                                          self.channel.name, self.resolution,
                                          self.job.id)

                # Create the upload queue
                upload_queue = self.create_upload_queue()
                self.job.upload_queue = upload_queue.url

                # Create the ingest queue
                ingest_queue = self.create_ingest_queue()
                self.job.ingest_queue = ingest_queue.url

                # Call the step function to populate the queue.
                self.job.step_function_arn = self.populate_upload_queue()

                # Compute # of tiles in the job
                x_extent = self.job.x_stop - self.job.x_start
                y_extent = self.job.y_stop - self.job.y_start
                z_extent = self.job.z_stop - self.job.z_start
                t_extent = self.job.t_stop - self.job.t_start
                num_tiles_in_x = math.ceil(x_extent / self.job.tile_size_x)
                num_tiles_in_y = math.ceil(y_extent / self.job.tile_size_y)
                num_tiles_in_z = math.ceil(z_extent / self.job.tile_size_z)
                num_tiles_in_t = math.ceil(t_extent / self.job.tile_size_t)
                self.job.tile_count = num_tiles_in_x * num_tiles_in_y * num_tiles_in_z * num_tiles_in_t
                self.job.save()

                # tile_bucket = TileBucket(self.job.collection + '&' + self.job.experiment)
                # self.create_ingest_credentials(upload_queue, tile_bucket)

        except BossError as err:
            raise BossError(err.message, err.error_code)
        except Exception as e:
            raise BossError(
                "Unable to create the upload and ingest queue.{}".format(e),
                ErrorCodes.BOSS_SYSTEM_ERROR)
        return self.job