class Interface(object):
def __init__(self, dataset_name, project_name, host_name=HOST_NAME):
# self.resource_interface = ResourceInterface(dataset_name, project_name, host_name)
self.cuboid_bucket = CuboidBucket(project_name)
self.cuboidindex_db = CuboidIndexDB(project_name)
# def deleteToken(self):
# """Delete the Token"""
# self.resource_interface.deleteToken()
# print 'Delete successful for token {}'.format(self.token)
def deleteProject(self):
"""Delete the project"""
return NotImplemented
# delete the project from s3 and dynamo
# self.s3_projdb.deleteNDProject()
# deleting the meta-data via resource interface
# self.resource_interface.deleteToken()
# self.resource_interface.deleteProject()
# print 'Delete successful for project {}'.format(self.project_name)
def deleteChannel(self, channel_name):
"""Delete the channel"""
try:
for item in self.cuboidindex_db.queryChannelItems(channel_name):
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except Exception as e:
print (e)
# delete the channel from s3 and dynamo
# self.s3_projdb.deleteNDChannel(channel_name)
# deleting the meta-data via resource interface
# self.resource_interface.deleteChannel(channel_name)
# print 'Delete successful for channel {}'.format(channel_name)
def deleteResolution(self, channel_name, resolution):
"""Delete an existing resolution"""
try:
for item in self.cuboidindex_db.queryResolutionItems(channel_name, resolution):
print(item['supercuboid_key'])
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except Exception as e:
print (e)
class S3ProjectDB:
"""Database for the projects"""
def __init__(self, pr):
"""Create the database connection"""
self.pr = pr
# create connections for cuboid bucket and cuboid dyanmo table
self.cuboid_bucket = CuboidBucket(self.pr.project_name, endpoint_url=ndingest_settings.S3_ENDPOINT)
self.cuboidindex_db = CuboidIndexDB(self.pr.project_name, endpoint_url=ndingest_settings.DYNAMO_ENDPOINT)
def __del__(self):
"""Close the database connection"""
self.close()
def close (self):
"""Close the database connection"""
pass
def newNDProject(self):
"""Create the database for a project."""
pass
def newNDChannel(self, channel_name):
"""Create the tables for a channel."""
pass
def deleteNDProject(self):
"""Delete a project in s3 and dyanmo"""
try:
for item in self.cuboidindex_db.queryProjectItems():
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error("Error in deleting S3 project {}. {}".format(self.pr.project_name, e))
raise NDWSError("Error in deleting S3 project {}. {}".format(self.pr.project_name, e))
def deleteNDChannel(self, channel_name):
"""Delete a channel in s3 and dynamo"""
try:
for item in self.cuboidindex_db.queryChannelItems(channel_name):
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error("Error in deleting S3 channel {}. {}".format(channel_name, e))
raise NDWSError("Error in deleting S3 channel {}. {}".format(channel_name, e))
def deleteNDResolution(self, channel_name, resolution):
"""Delete the resolution in s3 and dynamo"""
try:
for item in self.cuboidindex_db.queryResolutionItems(channel_name, resolution):
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error("Error in deleting S3 channel resolution {},{}. {}".format(channel_name, resolution, e))
raise NDWSError("Error in deleting S3 channel resolution {},{}. {}".format(channel_name, resolution, e))
class S3ProjectDB:
"""Database for the projects"""
def __init__(self, pr):
"""Create the database connection"""
self.pr = pr
# create connections for cuboid bucket and cuboid dyanmo table
self.cuboid_bucket = CuboidBucket(
self.pr.project_name, endpoint_url=ndingest_settings.S3_ENDPOINT)
self.cuboidindex_db = CuboidIndexDB(
self.pr.project_name,
endpoint_url=ndingest_settings.DYNAMO_ENDPOINT)
def __del__(self):
"""Close the database connection"""
self.close()
def close(self):
"""Close the database connection"""
pass
def newNDProject(self):
"""Create the database for a project."""
pass
def newNDChannel(self, channel_name):
"""Create the tables for a channel."""
pass
def deleteNDProject(self):
"""Delete a project in s3 and dyanmo"""
try:
for item in self.cuboidindex_db.queryProjectItems():
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error("Error in deleting S3 project {}. {}".format(
self.pr.project_name, e))
raise NDWSError("Error in deleting S3 project {}. {}".format(
self.pr.project_name, e))
def deleteNDChannel(self, channel_name):
"""Delete a channel in s3 and dynamo"""
try:
for item in self.cuboidindex_db.queryChannelItems(channel_name):
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error("Error in deleting S3 channel {}. {}".format(
channel_name, e))
raise NDWSError("Error in deleting S3 channel {}. {}".format(
channel_name, e))
def deleteNDResolution(self, channel_name, resolution):
"""Delete the resolution in s3 and dynamo"""
try:
for item in self.cuboidindex_db.queryResolutionItems(
channel_name, resolution):
self.cuboid_bucket.deleteObject(item['supercuboid_key'])
self.cuboidindex_db.deleteItem(item['supercuboid_key'])
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
logger.warning("Resource was not accessible {}".format(e))
pass
else:
raise e
except Exception as e:
logger.error(
"Error in deleting S3 channel resolution {},{}. {}".format(
channel_name, resolution, e))
raise NDWSError(
"Error in deleting S3 channel resolution {},{}. {}".format(
channel_name, resolution, e))
class Test_CuboidIndexDB():
def setup_class(self):
"""Setup parameters"""
try:
CuboidIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT)
except Exception as e:
pass
self.cuboid_index = CuboidIndexDB(
nd_proj.project_name, endpoint_url=settings.DYNAMO_ENDPOINT)
def teardown_class(self):
"""Teardown parameters"""
CuboidIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT)
def test_putItem(self):
"""Test data insertion"""
# inserting three values for task 0, zvalues 0-2
x_value = 0
y_value = 0
for z_value in range(0, 2, 1):
self.cuboid_index.putItem(nd_proj.channel_name, nd_proj.resolution,
x_value, y_value, z_value)
# checking if the items were inserted
for z_value in range(0, 2, 1):
item_value = self.cuboid_index.getItem(nd_proj.channel_name,
nd_proj.resolution, x_value,
y_value, z_value)
assert (item_value['project_name'] == nd_proj.project_name)
assert (
item_value['channel_resolution_taskid'] == '{}&{}&{}'.format(
nd_proj.channel_name, nd_proj.resolution, 0))
# inserting two values for task 1, zvalues 0-1
for z_value in range(0, 1, 1):
self.cuboid_index.putItem(nd_proj.channel_name,
nd_proj.resolution,
x_value,
y_value,
z_value,
task_id=1)
# checking if the items were updated
for z_value in range(0, 1, 1):
item_value = self.cuboid_index.getItem(nd_proj.channel_name,
nd_proj.resolution, x_value,
y_value, z_value)
assert (item_value['project_name'] == nd_proj.project_name)
assert (
item_value['channel_resolution_taskid'] == '{}&{}&{}'.format(
nd_proj.channel_name, nd_proj.resolution, 1))
def test_queryProjectItems(self):
"""Test the query over SI"""
# inserting three values for task 0, zvalues 0-2
x_value = 0
y_value = 0
for z_value in range(0, 2, 1):
self.cuboid_index.putItem(nd_proj.channel_name, nd_proj.resolution,
x_value, y_value, z_value)
for item in self.cuboid_index.queryProjectItems():
assert (item['project_name'] == nd_proj.project_name)
for item in self.cuboid_index.queryChannelItems(nd_proj2.channel_name):
assert (item['channel_resolution_taskid'] == '{}&{}&{}'.format(
nd_proj2.channel_name, nd_proj.resolution, 0))
for item in self.cuboid_index.queryTaskItems(nd_proj.channel_name,
nd_proj.resolution, 1):
assert (item['channel_resolution_taskid'] == '{}&{}&{}'.format(
nd_proj2.channel_name, nd_proj.resolution, 0))
def test_deleteXYZ(self):
"""Test item deletion"""
x_value = 0
y_value = 0
for z_value in range(0, 2, 1):
value = self.cuboid_index.deleteXYZ(nd_proj.channel_name,
nd_proj.resolution, x_value,
y_value, z_value)
item = self.cuboid_index.getItem(nd_proj.channel_name,
nd_proj.resolution, x_value,
y_value, z_value)
assert (item == None)