def createIngestJob(self, user_id, config_data): """Create an ingest job based on the posted config data""" config_data = json.loads(config_data) # validate schema if self.validateConfig(config_data): try: # create the upload queue UploadQueue.createQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.upload_queue = UploadQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.ingest_job.upload_queue = self.upload_queue.url # create the ingest queue IngestQueue.createQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.ingest_job.ingest_queue = IngestQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT).url # create the cleanup queue CleanupQueue.createQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.ingest_job.cleanup_queue = CleanupQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT).url self.generateUploadTasks() self.ingest_job.user_id = user_id self.ingest_job.save() return NDIngestJob.serialize(self.ingest_job._job) except Exception as e: print(e) raise NDWSError(e)
def setup_class(self): """Setup class parameters""" if 'SQS_ENDPOINT' in dir(settings): self.endpoint_url = settings.SQS_ENDPOINT else: self.endpoint_url = None IngestQueue.createQueue(nd_proj, endpoint_url=self.endpoint_url) self.ingest_queue = IngestQueue(nd_proj, endpoint_url=self.endpoint_url)
def create_ingest_queue(self): """ Returns: """ IngestQueue.createQueue(self.nd_proj, endpoint_url=None) queue = IngestQueue(self.nd_proj, endpoint_url=None) return queue
def create_ingest_queue(self): """ Create an ingest queue for an ingest job using the ndingest library Returns: IngestQueue : Returns a ingest queue object """ IngestQueue.createQueue(self.nd_proj, endpoint_url=None) queue = IngestQueue(self.nd_proj, endpoint_url=None) return queue
def create_ingest_queue(self): """ Create an ingest queue for an ingest job using the ndingest library Returns: IngestQueue : Returns a ingest queue object """ IngestQueue.createQueue(self.nd_proj, endpoint_url=None) queue = IngestQueue(self.nd_proj, endpoint_url=None) return queue
def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url="http://localhost:8000") CuboidIndexDB.createTable(endpoint_url="http://localhost:8000") except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url="http://localhost:8000") # create the tile bucket TileBucket.createBucket(endpoint_url="http://localhost:4567") self.tile_bucket = TileBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") self.tiles = [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] message_id = "testing" receipt_handle = "123456" # insert SUPER_CUBOID_SIZE tiles in the bucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_handle = cStringIO.StringIO() self.tile_bucket.putObject( tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, message_id, receipt_handle, ) # creating the cuboid bucket CuboidBucket.createBucket(endpoint_url="http://localhost:4567") self.cuboid_bucket = CuboidBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") # create the ingest queue IngestQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568") self.ingest_queue = IngestQueue(nd_proj, endpoint_url="http://localhost:4568") # send message to the ingest queue morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, morton_index) response = self.ingest_queue.sendMessage(supercuboid_key) # create the cleanup queue CleanupQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568")
def create_ingest_queue(self): """ Create an ingest queue for an ingest job using the ndingest library Returns: IngestQueue : Returns a ingest queue object """ IngestQueue.createQueue(self.nd_proj, endpoint_url=None) queue = IngestQueue(self.nd_proj, endpoint_url=None) timeout = self.get_ingest_lambda_timeout(INGEST_LAMBDA) # Ensure visibility timeout is greater than the ingest lambda that pulls # from it with a bit of buffer. queue.queue.set_attributes(Attributes={'VisibilityTimeout': str(timeout + 20)}) return queue
def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT) except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url=settings.DYNAMO_ENDPOINT) # create the ingest queue IngestQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) # create the upload queue UploadQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) self.upload_queue = UploadQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) tile_bucket = TileBucket(nd_proj.project_name, endpoint_url=settings.S3_ENDPOINT) [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] message = serializer.encodeUploadMessage( nd_proj.project_name, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, self.z_tile, ) # insert message in the upload queue self.upload_queue.sendMessage(message) # receive message and upload object for ( message_id, receipt_handle, message_body, ) in self.upload_queue.receiveMessage(): tile_handle = cStringIO.StringIO() tile_bucket.putObject( tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, self.z_tile, message_id, receipt_handle, )