def teardown_class(self): """Teardown class parameters""" # cleanup tilebucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_key = self.tile_bucket.encodeObjectKey( nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, ) self.tile_bucket.deleteObject(tile_key) morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, self.tiles) self.cuboid_bucket.deleteObject(supercuboid_key) # delete created entities TileIndexDB.deleteTable(endpoint_url="http://localhost:8000") CuboidIndexDB.deleteTable(endpoint_url="http://localhost:8000") IngestQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") CleanupQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") TileBucket.deleteBucket(endpoint_url="http://localhost:4567") try: CuboidBucket.deleteBucket(endpoint_url="http://localhost:4567") except Exception as e: pass
def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT) CuboidIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT) except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url=settings.DYNAMO_ENDPOINT) self.tile_bucket = TileBucket(nd_proj.project_name, endpoint_url=settings.S3_ENDPOINT) [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] supercuboid_key = 'testing' message_id = '123456' receipt_handle = 'testing123456' message = serializer.encodeDeleteMessage(supercuboid_key, message_id, receipt_handle) # insert message in the upload queue CleanupQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) self.cleanup_queue = CleanupQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) self.cleanup_queue.sendMessage(message) # receive message and upload object for z_index in range(self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_handle = cStringIO.StringIO() self.tile_bucket.putObject(tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, message_id, receipt_handle)
def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url="http://localhost:8000") CuboidIndexDB.createTable(endpoint_url="http://localhost:8000") except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url="http://localhost:8000") # create the tile bucket TileBucket.createBucket(endpoint_url="http://localhost:4567") self.tile_bucket = TileBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") self.tiles = [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] message_id = "testing" receipt_handle = "123456" # insert SUPER_CUBOID_SIZE tiles in the bucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_handle = cStringIO.StringIO() self.tile_bucket.putObject( tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, message_id, receipt_handle, ) # creating the cuboid bucket CuboidBucket.createBucket(endpoint_url="http://localhost:4567") self.cuboid_bucket = CuboidBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") # create the ingest queue IngestQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568") self.ingest_queue = IngestQueue(nd_proj, endpoint_url="http://localhost:4568") # send message to the ingest queue morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, morton_index) response = self.ingest_queue.sendMessage(supercuboid_key) # create the cleanup queue CleanupQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568")
def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT) except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url=settings.DYNAMO_ENDPOINT) # create the ingest queue IngestQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) # create the upload queue UploadQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) self.upload_queue = UploadQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) tile_bucket = TileBucket(nd_proj.project_name, endpoint_url=settings.S3_ENDPOINT) [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] message = serializer.encodeUploadMessage( nd_proj.project_name, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, self.z_tile, ) # insert message in the upload queue self.upload_queue.sendMessage(message) # receive message and upload object for ( message_id, receipt_handle, message_body, ) in self.upload_queue.receiveMessage(): tile_handle = cStringIO.StringIO() tile_bucket.putObject( tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, self.z_tile, message_id, receipt_handle, )
def teardown_class(self): """Teardown class parameters""" TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT) IngestQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) UploadQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
class Test_UploadLambda: def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT) except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url=settings.DYNAMO_ENDPOINT) # create the ingest queue IngestQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) # create the upload queue UploadQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) self.upload_queue = UploadQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) tile_bucket = TileBucket(nd_proj.project_name, endpoint_url=settings.S3_ENDPOINT) [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] message = serializer.encodeUploadMessage( nd_proj.project_name, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, self.z_tile, ) # insert message in the upload queue self.upload_queue.sendMessage(message) # receive message and upload object for ( message_id, receipt_handle, message_body, ) in self.upload_queue.receiveMessage(): tile_handle = cStringIO.StringIO() tile_bucket.putObject( tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, self.z_tile, message_id, receipt_handle, ) def teardown_class(self): """Teardown class parameters""" TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT) IngestQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) UploadQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) def test_Uploadevent(self): """Testing the event""" # creating an emulambda function func = emulambda.import_lambda("uploadlambda.lambda_handler") # creating an emulambda event event = emulambda.parse_event( open("../ndlambda/functions/upload/upload_event.json").read()) # calling the emulambda function to invoke a lambda emulambda.invoke_lambda(func, event, None, 0, None) # testing if the index was updated in tileindexdb supercuboid_key = self.tileindex_db.generatePrimaryKey( nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, self.z_tile, ) item = self.tileindex_db.getItem(supercuboid_key) assert item["zindex_list"] == set([0]) # testing if the message was deleted from the upload queue or not for message in self.upload_queue.receiveMessage(): assert False
def teardown_class(self): """Teardown class parameters""" TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT) CuboidIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT) CleanupQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
class Test_UploadLambda: def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT) CuboidIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT) except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url=settings.DYNAMO_ENDPOINT) self.tile_bucket = TileBucket(nd_proj.project_name, endpoint_url=settings.S3_ENDPOINT) [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] supercuboid_key = 'testing' message_id = '123456' receipt_handle = 'testing123456' message = serializer.encodeDeleteMessage(supercuboid_key, message_id, receipt_handle) # insert message in the upload queue CleanupQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) self.cleanup_queue = CleanupQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) self.cleanup_queue.sendMessage(message) # receive message and upload object for z_index in range(self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_handle = cStringIO.StringIO() self.tile_bucket.putObject(tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, message_id, receipt_handle) def teardown_class(self): """Teardown class parameters""" TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT) CuboidIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT) CleanupQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) def test_Uploadevent(self): """Testing the event""" # creating an emulambda function func = emulambda.import_lambda('cleanuplambda.lambda_handler') # creating an emulambda event event = emulambda.parse_event( open('../ndlambda/functions/cleanup/cleanup_event.json').read()) # calling the emulambda function to invoke a lambda emulambda.invoke_lambda(func, event, None, 0, None) # test if there are any tiles leftover in tile bucket for z_index in range(self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile = self.tile_bucket.getObject(nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index) assert (tile is None) # check if there are any entires left in the tileindex table supercuboid_key = self.tileindex_db.generatePrimaryKey( nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, self.z_tile) item = self.tileindex_db.getItem(supercuboid_key) assert (item is None) # testing if the message was deleted from the cleanup queue or not for message in self.cleanup_queue.receiveMessage(): # KL TODO write the message id into the JSON event file directly print message
def teardown_class(self): """Teardown parameters""" TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT)
def setup_class(self): """Setup parameters""" TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT) self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url=settings.DYNAMO_ENDPOINT)
class Test_TileIndexDB: def setup_class(self): """Setup parameters""" TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT) self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url=settings.DYNAMO_ENDPOINT) def teardown_class(self): """Teardown parameters""" TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT) def test_putItem(self): """Test data insertion""" x_tile = 0 y_tile = 0 # inserting three values for task 0 for z_tile in range(0, 3, 1): self.tileindex_db.putItem( nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile, z_tile, task_id=0, ) # inserting 2 values for task 1 for z_tile in range(66, 68, 1): self.tileindex_db.putItem( nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile, z_tile, task_id=1, ) # checking if the items were inserted z_tile = 0 supercuboid_key = self.tileindex_db.generatePrimaryKey( nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile, z_tile) item_value = self.tileindex_db.getItem(supercuboid_key) assert item_value["zindex_list"] == set([0, 1, 2]) z_tile = 65 supercuboid_key = self.tileindex_db.generatePrimaryKey( nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile, z_tile) item_value = self.tileindex_db.getItem(supercuboid_key) assert item_value["zindex_list"] == set([66, 67]) def test_queryTaskItems(self): """Test the query over SI""" for item in self.tileindex_db.getTaskItems(0): assert item["zindex_list"] == set([0, 1, 2]) def test_supercuboidReady(self): """Test if the supercuboid is ready""" x_tile = 0 y_tile = 0 for z_tile in range(129, 129 + settings.SUPER_CUBOID_SIZE[2], 1): supercuboid_key, supercuboid_ready = self.tileindex_db.putItem( nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile, z_tile, task_id=0, ) if z_tile < 129 + settings.SUPER_CUBOID_SIZE[2]: assert supercuboid_ready is False else: assert supercuboid_ready is True def test_deleteItem(self): """Test item deletion""" x_tile = 0 y_tile = 0 # inserting three values for task 0 for z_tile in range(0, 3, 1): supercuboid_key = self.tileindex_db.generatePrimaryKey( nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile, z_tile) self.tileindex_db.deleteItem(supercuboid_key) # inserting 2 values for task 1 for z_tile in range(66, 68, 1): supercuboid_key = self.tileindex_db.generatePrimaryKey( nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile, z_tile) self.tileindex_db.deleteItem(supercuboid_key) # inserting three values for task 0 for z_tile in range(0, 3, 1): supercuboid_key = self.tileindex_db.generatePrimaryKey( nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile, z_tile) item = self.tileindex_db.getItem(supercuboid_key) assert item == None # inserting 2 values for task 1 for z_tile in range(66, 68, 1): supercuboid_key = self.tileindex_db.generatePrimaryKey( nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile, z_tile) item = self.tileindex_db.getItem(supercuboid_key) assert item == None