def delete_tiles(self, ingest_job): """ Delete all remaining tiles from the tile index database and tile bucket Args: ingest_job: Ingest job model Returns: None Raises: BossError : For exceptions that happen while deleting the tiles and index """ try: # Get all the chunks for a job tiledb = BossTileIndexDB(ingest_job.collection + '&' + ingest_job.experiment) tilebucket = TileBucket(ingest_job.collection + '&' + ingest_job.experiment) chunks = list(tiledb.getTaskItems(ingest_job.id)) for chunk in chunks: chunk_key = chunk['chunk_key'] # delete each tile in the chunk for key in chunk['tile_uploaded_map']: response = tilebucket.deleteObject(key) tiledb.deleteCuboid(chunk['chunk_key'], ingest_job.id) except Exception as e: raise BossError( "Exception while deleteing tiles for the ingest job {}. {}". format(ingest_job.id, e), ErrorCodes.BOSS_SYSTEM_ERROR)
def delete_tiles(self, ingest_job): """ Delete all remaining tiles from the tile index database and tile bucket 5/24/2018 - This code depends on a GSI for the tile index. The GSI was removed because its key didn't shard well. Cleanup will now be handled by TTL policies applied to the tile bucket and the tile index. This method will be removed once that code is merged. Args: ingest_job: Ingest job model Returns: None Raises: BossError : For exceptions that happen while deleting the tiles and index """ try: # Get all the chunks for a job tiledb = BossTileIndexDB(ingest_job.collection + '&' + ingest_job.experiment) tilebucket = TileBucket(ingest_job.collection + '&' + ingest_job.experiment) chunks = list(tiledb.getTaskItems(ingest_job.id)) for chunk in chunks: # delete each tile in the chunk for key in chunk['tile_uploaded_map']: response = tilebucket.deleteObject(key) tiledb.deleteCuboid(chunk['chunk_key'], ingest_job.id) except Exception as e: raise BossError("Exception while deleteing tiles for the ingest job {}. {}".format(ingest_job.id, e), ErrorCodes.BOSS_SYSTEM_ERROR)
class Test_IngestLambda: def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url="http://localhost:8000") CuboidIndexDB.createTable(endpoint_url="http://localhost:8000") except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url="http://localhost:8000") # create the tile bucket TileBucket.createBucket(endpoint_url="http://localhost:4567") self.tile_bucket = TileBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") self.tiles = [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] message_id = "testing" receipt_handle = "123456" # insert SUPER_CUBOID_SIZE tiles in the bucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_handle = cStringIO.StringIO() self.tile_bucket.putObject( tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, message_id, receipt_handle, ) # creating the cuboid bucket CuboidBucket.createBucket(endpoint_url="http://localhost:4567") self.cuboid_bucket = CuboidBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") # create the ingest queue IngestQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568") self.ingest_queue = IngestQueue(nd_proj, endpoint_url="http://localhost:4568") # send message to the ingest queue morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, morton_index) response = self.ingest_queue.sendMessage(supercuboid_key) # create the cleanup queue CleanupQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568") def teardown_class(self): """Teardown class parameters""" # cleanup tilebucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_key = self.tile_bucket.encodeObjectKey( nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, ) self.tile_bucket.deleteObject(tile_key) morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, self.tiles) self.cuboid_bucket.deleteObject(supercuboid_key) # delete created entities TileIndexDB.deleteTable(endpoint_url="http://localhost:8000") CuboidIndexDB.deleteTable(endpoint_url="http://localhost:8000") IngestQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") CleanupQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") TileBucket.deleteBucket(endpoint_url="http://localhost:4567") try: CuboidBucket.deleteBucket(endpoint_url="http://localhost:4567") except Exception as e: pass def test_Uploadevent(self): """Testing the event""" # creating an emulambda function func = emulambda.import_lambda("ingestlambda.lambda_handler") # creating an emulambda event event = emulambda.parse_event( open("../ndlambda/functions/ingest/ingest_event.json").read()) # calling the emulambda function to invoke a lambda emulambda.invoke_lambda(func, event, None, 0, None) # testing if the supercuboid was inserted in the bucket morton_index = XYZMorton(self.tiles) cuboid = self.cuboid_bucket.getObject(nd_proj.channel_name, nd_proj.resolution, morton_index) # testing if the message was removed from the ingest queue for message in self.ingest_queue.receiveMessage(): # KL TODO write the message id into the JSON event file directly print(message)
resource.data['channel']['name']) sns_client.publish( TopicArn=topic_arn, Subject='Object services misuse', Message=msg) # Delete message since it was processed successfully ingest_queue.deleteMessage(msg_id, msg_rx_handle) # Delete Tiles for tile in tile_key_list: for try_cnt in range(0, 4): try: time.sleep(try_cnt) print("Deleting tile: {}".format(tile)) tile_bucket.deleteObject(tile) break except: print("failed") # Delete Entry in tile table for try_cnt in range(0, 4): try: time.sleep(try_cnt) tile_index_db.deleteCuboid(chunk_key, int(msg_data["ingest_job"])) break except: print("failed") # Increment run counter run_cnt += 1