def teardown_class(self):
"""Teardown class parameters"""
# cleanup tilebucket
for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1):
tile_key = self.tile_bucket.encodeObjectKey(
nd_proj.channel_name,
nd_proj.resolution,
self.x_tile,
self.y_tile,
z_index,
)
self.tile_bucket.deleteObject(tile_key)
morton_index = XYZMorton(self.tiles)
supercuboid_key = self.cuboid_bucket.generateSupercuboidKey(
nd_proj.channel_name, nd_proj.resolution, self.tiles)
self.cuboid_bucket.deleteObject(supercuboid_key)
# delete created entities
TileIndexDB.deleteTable(endpoint_url="http://localhost:8000")
CuboidIndexDB.deleteTable(endpoint_url="http://localhost:8000")
IngestQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568")
CleanupQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568")
TileBucket.deleteBucket(endpoint_url="http://localhost:4567")
try:
CuboidBucket.deleteBucket(endpoint_url="http://localhost:4567")
except Exception as e:
pass
def setup_class(self):
"""Setup class parameters"""
# create the tile index table. skip if it exists
try:
TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT)
CuboidIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT)
except Exception as e:
pass
self.tileindex_db = TileIndexDB(nd_proj.project_name,
endpoint_url=settings.DYNAMO_ENDPOINT)
self.tile_bucket = TileBucket(nd_proj.project_name,
endpoint_url=settings.S3_ENDPOINT)
[self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0]
supercuboid_key = 'testing'
message_id = '123456'
receipt_handle = 'testing123456'
message = serializer.encodeDeleteMessage(supercuboid_key, message_id,
receipt_handle)
# insert message in the upload queue
CleanupQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
self.cleanup_queue = CleanupQueue(nd_proj,
endpoint_url=settings.SQS_ENDPOINT)
self.cleanup_queue.sendMessage(message)
# receive message and upload object
for z_index in range(self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1):
tile_handle = cStringIO.StringIO()
self.tile_bucket.putObject(tile_handle, nd_proj.channel_name,
nd_proj.resolution, self.x_tile,
self.y_tile, z_index, message_id,
receipt_handle)
def setup_class(self):
"""Setup class parameters"""
# create the tile index table. skip if it exists
try:
TileIndexDB.createTable(endpoint_url="http://localhost:8000")
CuboidIndexDB.createTable(endpoint_url="http://localhost:8000")
except Exception as e:
pass
self.tileindex_db = TileIndexDB(nd_proj.project_name,
endpoint_url="http://localhost:8000")
# create the tile bucket
TileBucket.createBucket(endpoint_url="http://localhost:4567")
self.tile_bucket = TileBucket(nd_proj.project_name,
endpoint_url="http://localhost:4567")
self.tiles = [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0]
message_id = "testing"
receipt_handle = "123456"
# insert SUPER_CUBOID_SIZE tiles in the bucket
for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1):
tile_handle = cStringIO.StringIO()
self.tile_bucket.putObject(
tile_handle,
nd_proj.channel_name,
nd_proj.resolution,
self.x_tile,
self.y_tile,
z_index,
message_id,
receipt_handle,
)
# creating the cuboid bucket
CuboidBucket.createBucket(endpoint_url="http://localhost:4567")
self.cuboid_bucket = CuboidBucket(nd_proj.project_name,
endpoint_url="http://localhost:4567")
# create the ingest queue
IngestQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568")
self.ingest_queue = IngestQueue(nd_proj,
endpoint_url="http://localhost:4568")
# send message to the ingest queue
morton_index = XYZMorton(self.tiles)
supercuboid_key = self.cuboid_bucket.generateSupercuboidKey(
nd_proj.channel_name, nd_proj.resolution, morton_index)
response = self.ingest_queue.sendMessage(supercuboid_key)
# create the cleanup queue
CleanupQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568")
def setup_class(self):
"""Setup class parameters"""
# create the tile index table. skip if it exists
try:
TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT)
except Exception as e:
pass
self.tileindex_db = TileIndexDB(nd_proj.project_name,
endpoint_url=settings.DYNAMO_ENDPOINT)
# create the ingest queue
IngestQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
# create the upload queue
UploadQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
self.upload_queue = UploadQueue(nd_proj,
endpoint_url=settings.SQS_ENDPOINT)
tile_bucket = TileBucket(nd_proj.project_name,
endpoint_url=settings.S3_ENDPOINT)
[self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0]
message = serializer.encodeUploadMessage(
nd_proj.project_name,
nd_proj.channel_name,
nd_proj.resolution,
self.x_tile,
self.y_tile,
self.z_tile,
)
# insert message in the upload queue
self.upload_queue.sendMessage(message)
# receive message and upload object
for (
message_id,
receipt_handle,
message_body,
) in self.upload_queue.receiveMessage():
tile_handle = cStringIO.StringIO()
tile_bucket.putObject(
tile_handle,
nd_proj.channel_name,
nd_proj.resolution,
self.x_tile,
self.y_tile,
self.z_tile,
message_id,
receipt_handle,
)
def teardown_class(self):
"""Teardown class parameters"""
TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT)
IngestQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
UploadQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
class Test_UploadLambda:
def setup_class(self):
"""Setup class parameters"""
# create the tile index table. skip if it exists
try:
TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT)
except Exception as e:
pass
self.tileindex_db = TileIndexDB(nd_proj.project_name,
endpoint_url=settings.DYNAMO_ENDPOINT)
# create the ingest queue
IngestQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
# create the upload queue
UploadQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
self.upload_queue = UploadQueue(nd_proj,
endpoint_url=settings.SQS_ENDPOINT)
tile_bucket = TileBucket(nd_proj.project_name,
endpoint_url=settings.S3_ENDPOINT)
[self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0]
message = serializer.encodeUploadMessage(
nd_proj.project_name,
nd_proj.channel_name,
nd_proj.resolution,
self.x_tile,
self.y_tile,
self.z_tile,
)
# insert message in the upload queue
self.upload_queue.sendMessage(message)
# receive message and upload object
for (
message_id,
receipt_handle,
message_body,
) in self.upload_queue.receiveMessage():
tile_handle = cStringIO.StringIO()
tile_bucket.putObject(
tile_handle,
nd_proj.channel_name,
nd_proj.resolution,
self.x_tile,
self.y_tile,
self.z_tile,
message_id,
receipt_handle,
)
def teardown_class(self):
"""Teardown class parameters"""
TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT)
IngestQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
UploadQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
def test_Uploadevent(self):
"""Testing the event"""
# creating an emulambda function
func = emulambda.import_lambda("uploadlambda.lambda_handler")
# creating an emulambda event
event = emulambda.parse_event(
open("../ndlambda/functions/upload/upload_event.json").read())
# calling the emulambda function to invoke a lambda
emulambda.invoke_lambda(func, event, None, 0, None)
# testing if the index was updated in tileindexdb
supercuboid_key = self.tileindex_db.generatePrimaryKey(
nd_proj.channel_name,
nd_proj.resolution,
self.x_tile,
self.y_tile,
self.z_tile,
)
item = self.tileindex_db.getItem(supercuboid_key)
assert item["zindex_list"] == set([0])
# testing if the message was deleted from the upload queue or not
for message in self.upload_queue.receiveMessage():
assert False
def teardown_class(self):
"""Teardown class parameters"""
TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT)
CuboidIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT)
CleanupQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
class Test_UploadLambda:
def setup_class(self):
"""Setup class parameters"""
# create the tile index table. skip if it exists
try:
TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT)
CuboidIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT)
except Exception as e:
pass
self.tileindex_db = TileIndexDB(nd_proj.project_name,
endpoint_url=settings.DYNAMO_ENDPOINT)
self.tile_bucket = TileBucket(nd_proj.project_name,
endpoint_url=settings.S3_ENDPOINT)
[self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0]
supercuboid_key = 'testing'
message_id = '123456'
receipt_handle = 'testing123456'
message = serializer.encodeDeleteMessage(supercuboid_key, message_id,
receipt_handle)
# insert message in the upload queue
CleanupQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
self.cleanup_queue = CleanupQueue(nd_proj,
endpoint_url=settings.SQS_ENDPOINT)
self.cleanup_queue.sendMessage(message)
# receive message and upload object
for z_index in range(self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1):
tile_handle = cStringIO.StringIO()
self.tile_bucket.putObject(tile_handle, nd_proj.channel_name,
nd_proj.resolution, self.x_tile,
self.y_tile, z_index, message_id,
receipt_handle)
def teardown_class(self):
"""Teardown class parameters"""
TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT)
CuboidIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT)
CleanupQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
def test_Uploadevent(self):
"""Testing the event"""
# creating an emulambda function
func = emulambda.import_lambda('cleanuplambda.lambda_handler')
# creating an emulambda event
event = emulambda.parse_event(
open('../ndlambda/functions/cleanup/cleanup_event.json').read())
# calling the emulambda function to invoke a lambda
emulambda.invoke_lambda(func, event, None, 0, None)
# test if there are any tiles leftover in tile bucket
for z_index in range(self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1):
tile = self.tile_bucket.getObject(nd_proj.channel_name,
nd_proj.resolution, self.x_tile,
self.y_tile, z_index)
assert (tile is None)
# check if there are any entires left in the tileindex table
supercuboid_key = self.tileindex_db.generatePrimaryKey(
nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile,
self.z_tile)
item = self.tileindex_db.getItem(supercuboid_key)
assert (item is None)
# testing if the message was deleted from the cleanup queue or not
for message in self.cleanup_queue.receiveMessage():
# KL TODO write the message id into the JSON event file directly
print message
def teardown_class(self):
"""Teardown parameters"""
TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT)
def setup_class(self):
"""Setup parameters"""
TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT)
self.tileindex_db = TileIndexDB(nd_proj.project_name,
endpoint_url=settings.DYNAMO_ENDPOINT)
class Test_TileIndexDB:
def setup_class(self):
"""Setup parameters"""
TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT)
self.tileindex_db = TileIndexDB(nd_proj.project_name,
endpoint_url=settings.DYNAMO_ENDPOINT)
def teardown_class(self):
"""Teardown parameters"""
TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT)
def test_putItem(self):
"""Test data insertion"""
x_tile = 0
y_tile = 0
# inserting three values for task 0
for z_tile in range(0, 3, 1):
self.tileindex_db.putItem(
nd_proj.channel_name,
nd_proj.resolution,
x_tile,
y_tile,
z_tile,
task_id=0,
)
# inserting 2 values for task 1
for z_tile in range(66, 68, 1):
self.tileindex_db.putItem(
nd_proj.channel_name,
nd_proj.resolution,
x_tile,
y_tile,
z_tile,
task_id=1,
)
# checking if the items were inserted
z_tile = 0
supercuboid_key = self.tileindex_db.generatePrimaryKey(
nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile, z_tile)
item_value = self.tileindex_db.getItem(supercuboid_key)
assert item_value["zindex_list"] == set([0, 1, 2])
z_tile = 65
supercuboid_key = self.tileindex_db.generatePrimaryKey(
nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile, z_tile)
item_value = self.tileindex_db.getItem(supercuboid_key)
assert item_value["zindex_list"] == set([66, 67])
def test_queryTaskItems(self):
"""Test the query over SI"""
for item in self.tileindex_db.getTaskItems(0):
assert item["zindex_list"] == set([0, 1, 2])
def test_supercuboidReady(self):
"""Test if the supercuboid is ready"""
x_tile = 0
y_tile = 0
for z_tile in range(129, 129 + settings.SUPER_CUBOID_SIZE[2], 1):
supercuboid_key, supercuboid_ready = self.tileindex_db.putItem(
nd_proj.channel_name,
nd_proj.resolution,
x_tile,
y_tile,
z_tile,
task_id=0,
)
if z_tile < 129 + settings.SUPER_CUBOID_SIZE[2]:
assert supercuboid_ready is False
else:
assert supercuboid_ready is True
def test_deleteItem(self):
"""Test item deletion"""
x_tile = 0
y_tile = 0
# inserting three values for task 0
for z_tile in range(0, 3, 1):
supercuboid_key = self.tileindex_db.generatePrimaryKey(
nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile,
z_tile)
self.tileindex_db.deleteItem(supercuboid_key)
# inserting 2 values for task 1
for z_tile in range(66, 68, 1):
supercuboid_key = self.tileindex_db.generatePrimaryKey(
nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile,
z_tile)
self.tileindex_db.deleteItem(supercuboid_key)
# inserting three values for task 0
for z_tile in range(0, 3, 1):
supercuboid_key = self.tileindex_db.generatePrimaryKey(
nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile,
z_tile)
item = self.tileindex_db.getItem(supercuboid_key)
assert item == None
# inserting 2 values for task 1
for z_tile in range(66, 68, 1):
supercuboid_key = self.tileindex_db.generatePrimaryKey(
nd_proj.channel_name, nd_proj.resolution, x_tile, y_tile,
z_tile)
item = self.tileindex_db.getItem(supercuboid_key)
assert item == None