class Interface(object):
def __init__(self, dataset_name, project_name, host_name=HOST_NAME):
# self.resource_interface = ResourceInterface(dataset_name, project_name, host_name)
self.cuboid_bucket = CuboidBucket(project_name)
self.cuboidindex_db = CuboidIndexDB(project_name)
# def deleteToken(self):
# """Delete the Token"""
# self.resource_interface.deleteToken()
# print 'Delete successful for token {}'.format(self.token)
def deleteProject(self):
"""Delete the project"""
return NotImplemented
# delete the project from s3 and dynamo
# self.s3_projdb.deleteNDProject()
# deleting the meta-data via resource interface
# self.resource_interface.deleteToken()
# self.resource_interface.deleteProject()
# print 'Delete successful for project {}'.format(self.project_name)
def deleteChannel(self, channel_name):
"""Delete the channel"""
try:
for item in self.cuboidindex_db.queryChannelItems(channel_name):
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except Exception as e:
print (e)
# delete the channel from s3 and dynamo
# self.s3_projdb.deleteNDChannel(channel_name)
# deleting the meta-data via resource interface
# self.resource_interface.deleteChannel(channel_name)
# print 'Delete successful for channel {}'.format(channel_name)
def deleteResolution(self, channel_name, resolution):
"""Delete an existing resolution"""
try:
for item in self.cuboidindex_db.queryResolutionItems(channel_name, resolution):
print(item['supercuboid_key'])
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except Exception as e:
print (e)
class Test_Cuboid_Bucket:
def setup_class(self):
"""Setup Parameters"""
if "S3_ENDPOINT" in dir(settings):
self.endpoint_url = settings.S3_ENDPOINT
else:
self.endpoint_url = None
CuboidBucket.createBucket(endpoint_url=self.endpoint_url)
self.cuboid_bucket = CuboidBucket(
nd_proj.project_name, endpoint_url=self.endpoint_url
)
def teardown_class(self):
"""Teardown Parameters"""
# Ensure bucket empty before deleting.
for objs in self.cuboid_bucket.getAllObjects():
self.cuboid_bucket.deleteObject(objs.key)
CuboidBucket.deleteBucket(endpoint_url=self.endpoint_url)
@pytest.mark.skipif(
settings.PROJECT_NAME == "Boss", reason="putObject() not supported by the Boss"
)
def test_put_object(self):
"""Testing put object"""
cube_data = blosc.pack_array(np.zeros(settings.SUPER_CUBOID_SIZE))
for morton_index in range(0, 10, 1):
self.cuboid_bucket.putObject(
nd_proj.channel_name, nd_proj.resolution, morton_index, cube_data
)
for morton_index in range(0, 10, 1):
supercuboid_key = self.cuboid_bucket.generateSupercuboidKey(
nd_proj.channel_name, nd_proj.resolution, morton_index
)
self.cuboid_bucket.deleteObject(supercuboid_key)
def test_put_object_by_key(self):
hashm = hashlib.md5()
hashm.update(b"test_cuboidbucket_data")
cube_data = blosc.pack_array(np.zeros(settings.SUPER_CUBOID_SIZE))
for morton_index in range(0, 10, 1):
key = "{}&{}".format(hashm.hexdigest(), morton_index)
self.cuboid_bucket.putObjectByKey(key, cube_data)
for morton_index in range(0, 10, 1):
key = "{}&{}".format(hashm.hexdigest(), morton_index)
self.cuboid_bucket.deleteObject(key)
class S3ProjectDB:
"""Database for the projects"""
def __init__(self, pr):
"""Create the database connection"""
self.pr = pr
# create connections for cuboid bucket and cuboid dyanmo table
self.cuboid_bucket = CuboidBucket(
self.pr.project_name, endpoint_url=ndingest_settings.S3_ENDPOINT)
self.cuboidindex_db = CuboidIndexDB(
self.pr.project_name,
endpoint_url=ndingest_settings.DYNAMO_ENDPOINT)
def __del__(self):
"""Close the database connection"""
self.close()
def close(self):
"""Close the database connection"""
pass
def newNDProject(self):
"""Create the database for a project."""
pass
def newNDChannel(self, channel_name):
"""Create the tables for a channel."""
pass
def deleteNDProject(self):
"""Delete a project in s3 and dyanmo"""
try:
for item in self.cuboidindex_db.queryProjectItems():
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error("Error in deleting S3 project {}. {}".format(
self.pr.project_name, e))
raise NDWSError("Error in deleting S3 project {}. {}".format(
self.pr.project_name, e))
def deleteNDChannel(self, channel_name):
"""Delete a channel in s3 and dynamo"""
try:
for item in self.cuboidindex_db.queryChannelItems(channel_name):
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error("Error in deleting S3 channel {}. {}".format(
channel_name, e))
raise NDWSError("Error in deleting S3 channel {}. {}".format(
channel_name, e))
def deleteNDResolution(self, channel_name, resolution):
"""Delete the resolution in s3 and dynamo"""
try:
for item in self.cuboidindex_db.queryResolutionItems(
channel_name, resolution):
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error(
"Error in deleting S3 channel resolution {},{}. {}".format(
channel_name, resolution, e))
raise NDWSError(
"Error in deleting S3 channel resolution {},{}. {}".format(
channel_name, resolution, e))
class S3ProjectDB:
"""Database for the projects"""
def __init__(self, pr):
"""Create the database connection"""
self.pr = pr
# create connections for cuboid bucket and cuboid dyanmo table
self.cuboid_bucket = CuboidBucket(self.pr.project_name, endpoint_url=ndingest_settings.S3_ENDPOINT)
self.cuboidindex_db = CuboidIndexDB(self.pr.project_name, endpoint_url=ndingest_settings.DYNAMO_ENDPOINT)
def __del__(self):
"""Close the database connection"""
self.close()
def close (self):
"""Close the database connection"""
pass
def newNDProject(self):
"""Create the database for a project."""
pass
def newNDChannel(self, channel_name):
"""Create the tables for a channel."""
pass
def deleteNDProject(self):
"""Delete a project in s3 and dyanmo"""
try:
for item in self.cuboidindex_db.queryProjectItems():
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error("Error in deleting S3 project {}. {}".format(self.pr.project_name, e))
raise NDWSError("Error in deleting S3 project {}. {}".format(self.pr.project_name, e))
def deleteNDChannel(self, channel_name):
"""Delete a channel in s3 and dynamo"""
try:
for item in self.cuboidindex_db.queryChannelItems(channel_name):
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error("Error in deleting S3 channel {}. {}".format(channel_name, e))
raise NDWSError("Error in deleting S3 channel {}. {}".format(channel_name, e))
def deleteNDResolution(self, channel_name, resolution):
"""Delete the resolution in s3 and dynamo"""
try:
for item in self.cuboidindex_db.queryResolutionItems(channel_name, resolution):
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error("Error in deleting S3 channel resolution {},{}. {}".format(channel_name, resolution, e))
raise NDWSError("Error in deleting S3 channel resolution {},{}. {}".format(channel_name, resolution, e))
class Test_IngestLambda:
def setup_class(self):
"""Setup class parameters"""
# create the tile index table. skip if it exists
try:
TileIndexDB.createTable(endpoint_url="http://localhost:8000")
CuboidIndexDB.createTable(endpoint_url="http://localhost:8000")
except Exception as e:
pass
self.tileindex_db = TileIndexDB(nd_proj.project_name,
endpoint_url="http://localhost:8000")
# create the tile bucket
TileBucket.createBucket(endpoint_url="http://localhost:4567")
self.tile_bucket = TileBucket(nd_proj.project_name,
endpoint_url="http://localhost:4567")
self.tiles = [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0]
message_id = "testing"
receipt_handle = "123456"
# insert SUPER_CUBOID_SIZE tiles in the bucket
for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1):
tile_handle = cStringIO.StringIO()
self.tile_bucket.putObject(
tile_handle,
nd_proj.channel_name,
nd_proj.resolution,
self.x_tile,
self.y_tile,
z_index,
message_id,
receipt_handle,
)
# creating the cuboid bucket
CuboidBucket.createBucket(endpoint_url="http://localhost:4567")
self.cuboid_bucket = CuboidBucket(nd_proj.project_name,
endpoint_url="http://localhost:4567")
# create the ingest queue
IngestQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568")
self.ingest_queue = IngestQueue(nd_proj,
endpoint_url="http://localhost:4568")
# send message to the ingest queue
morton_index = XYZMorton(self.tiles)
supercuboid_key = self.cuboid_bucket.generateSupercuboidKey(
nd_proj.channel_name, nd_proj.resolution, morton_index)
response = self.ingest_queue.sendMessage(supercuboid_key)
# create the cleanup queue
CleanupQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568")
def teardown_class(self):
"""Teardown class parameters"""
# cleanup tilebucket
for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1):
tile_key = self.tile_bucket.encodeObjectKey(
nd_proj.channel_name,
nd_proj.resolution,
self.x_tile,
self.y_tile,
z_index,
)
self.tile_bucket.deleteObject(tile_key)
morton_index = XYZMorton(self.tiles)
supercuboid_key = self.cuboid_bucket.generateSupercuboidKey(
nd_proj.channel_name, nd_proj.resolution, self.tiles)
self.cuboid_bucket.deleteObject(supercuboid_key)
# delete created entities
TileIndexDB.deleteTable(endpoint_url="http://localhost:8000")
CuboidIndexDB.deleteTable(endpoint_url="http://localhost:8000")
IngestQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568")
CleanupQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568")
TileBucket.deleteBucket(endpoint_url="http://localhost:4567")
try:
CuboidBucket.deleteBucket(endpoint_url="http://localhost:4567")
except Exception as e:
pass
def test_Uploadevent(self):
"""Testing the event"""
# creating an emulambda function
func = emulambda.import_lambda("ingestlambda.lambda_handler")
# creating an emulambda event
event = emulambda.parse_event(
open("../ndlambda/functions/ingest/ingest_event.json").read())
# calling the emulambda function to invoke a lambda
emulambda.invoke_lambda(func, event, None, 0, None)
# testing if the supercuboid was inserted in the bucket
morton_index = XYZMorton(self.tiles)
cuboid = self.cuboid_bucket.getObject(nd_proj.channel_name,
nd_proj.resolution, morton_index)
# testing if the message was removed from the ingest queue
for message in self.ingest_queue.receiveMessage():
# KL TODO write the message id into the JSON event file directly
print(message)