def teardown_class(self): """Teardown class parameters""" # cleanup tilebucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_key = self.tile_bucket.encodeObjectKey( nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, ) self.tile_bucket.deleteObject(tile_key) morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, self.tiles) self.cuboid_bucket.deleteObject(supercuboid_key) # delete created entities TileIndexDB.deleteTable(endpoint_url="http://localhost:8000") CuboidIndexDB.deleteTable(endpoint_url="http://localhost:8000") IngestQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") CleanupQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") TileBucket.deleteBucket(endpoint_url="http://localhost:4567") try: CuboidBucket.deleteBucket(endpoint_url="http://localhost:4567") except Exception as e: pass
def teardown_class(self): """Teardown Parameters""" # Ensure bucket empty before deleting. for objs in self.cuboid_bucket.getAllObjects(): self.cuboid_bucket.deleteObject(objs.key) CuboidBucket.deleteBucket(endpoint_url=self.endpoint_url)
def __init__(self, pr): """Create the database connection""" self.pr = pr # create connections for cuboid bucket and cuboid dyanmo table self.cuboid_bucket = CuboidBucket( self.pr.project_name, endpoint_url=ndingest_settings.S3_ENDPOINT) self.cuboidindex_db = CuboidIndexDB( self.pr.project_name, endpoint_url=ndingest_settings.DYNAMO_ENDPOINT)
def setup_class(self): """Setup Parameters""" if 'S3_ENDPOINT' in dir(settings): self.endpoint_url = settings.S3_ENDPOINT else: self.endpoint_url = None CuboidBucket.createBucket(endpoint_url=self.endpoint_url) self.cuboid_bucket = CuboidBucket(nd_proj.project_name, endpoint_url=self.endpoint_url)
def __init__(self, token_name, host_name=HOST_NAME): # configuring the logger based on the dataset we are uploading self.logger = logging.getLogger(token_name) self.logger.setLevel(logging.INFO) fh = logging.FileHandler('{}_upload.log'.format(token_name)) self.logger.addHandler(fh) self.info_interface = InfoInterface(host_name, token_name) self.project_name = self.info_interface.project_name self.cuboidindex_db = CuboidIndexDB(self.project_name) self.cuboid_bucket = CuboidBucket(self.project_name)
def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url="http://localhost:8000") CuboidIndexDB.createTable(endpoint_url="http://localhost:8000") except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url="http://localhost:8000") # create the tile bucket TileBucket.createBucket(endpoint_url="http://localhost:4567") self.tile_bucket = TileBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") self.tiles = [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] message_id = "testing" receipt_handle = "123456" # insert SUPER_CUBOID_SIZE tiles in the bucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_handle = cStringIO.StringIO() self.tile_bucket.putObject( tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, message_id, receipt_handle, ) # creating the cuboid bucket CuboidBucket.createBucket(endpoint_url="http://localhost:4567") self.cuboid_bucket = CuboidBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") # create the ingest queue IngestQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568") self.ingest_queue = IngestQueue(nd_proj, endpoint_url="http://localhost:4568") # send message to the ingest queue morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, morton_index) response = self.ingest_queue.sendMessage(supercuboid_key) # create the cleanup queue CleanupQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568")
class Interface(object): def __init__(self, dataset_name, project_name, host_name=HOST_NAME): # self.resource_interface = ResourceInterface(dataset_name, project_name, host_name) self.cuboid_bucket = CuboidBucket(project_name) self.cuboidindex_db = CuboidIndexDB(project_name) # def deleteToken(self): # """Delete the Token""" # self.resource_interface.deleteToken() # print 'Delete successful for token {}'.format(self.token) def deleteProject(self): """Delete the project""" return NotImplemented # delete the project from s3 and dynamo # self.s3_projdb.deleteNDProject() # deleting the meta-data via resource interface # self.resource_interface.deleteToken() # self.resource_interface.deleteProject() # print 'Delete successful for project {}'.format(self.project_name) def deleteChannel(self, channel_name): """Delete the channel""" try: for item in self.cuboidindex_db.queryChannelItems(channel_name): self.cuboid_bucket.deleteObject(item['supercuboid_key']) self.cuboidindex_db.deleteItem(item['supercuboid_key']) except Exception as e: print (e) # delete the channel from s3 and dynamo # self.s3_projdb.deleteNDChannel(channel_name) # deleting the meta-data via resource interface # self.resource_interface.deleteChannel(channel_name) # print 'Delete successful for channel {}'.format(channel_name) def deleteResolution(self, channel_name, resolution): """Delete an existing resolution""" try: for item in self.cuboidindex_db.queryResolutionItems(channel_name, resolution): print(item['supercuboid_key']) self.cuboid_bucket.deleteObject(item['supercuboid_key']) self.cuboidindex_db.deleteItem(item['supercuboid_key']) except Exception as e: print (e)
def __init__(self, token_name, host_name=HOST_NAME): """Create the bucket and intialize values""" # configuring the logger based on the dataset we are uploading self.logger = logging.getLogger(token_name) self.logger.setLevel(logging.INFO) fh = logging.FileHandler('{}.log'.format(token_name)) self.logger.addHandler(fh) # setting up the project metadata self.info_interface = InfoInterface(host_name, token_name) # creating the resource interface to the remote server # self.resource_interface = ResourceInterface(self.info_interface.dataset_name, self.info_interface.project_name, host_name, logger=self.logger) # self.proj = self.resource_interface.getProject() # create the s3 I/O and index objects self.cuboidindex_db = CuboidIndexDB(self.info_interface.project_name) self.cuboid_bucket = CuboidBucket(self.info_interface.project_name)
def __init__(self, db): """Connect to the S3 backend""" try: self.db = db self.project_name = self.db.proj.project_name self.cuboidindex_db = CuboidIndexDB(self.project_name) self.cuboid_bucket = CuboidBucket(self.project_name) except Exception, e: logger.error("Cannot connect to S3 backend") raise SpatialDBError("Cannot connect to S3 backend")
def createTestDB ( project_name, channel_list=['unit_anno'], channel_type=ANNOTATION, channel_datatype=UINT32, public=PUBLIC_TRUE, ximagesize=10000, yimagesize=10000, zimagesize=1000, xvoxelres=4.0, yvoxelres=4.0, zvoxelres=3.0, scalingoption=ZSLICES, scalinglevels=5, readonly=READONLY_FALSE, propagate=NOT_PROPAGATED, window=[0,0], time=[0,15], default=False, nd_version=ND_VERSION, token_name='unittest', user='******', dataset_name="unittest", base_resolution=0): """Create a unit test data base on the specified sit and name""" # setting s3backend to true if Redis and creating s3 bucket and dynamo table if KV_ENGINE == REDIS: s3backend = S3_TRUE CuboidIndexDB.createTable() CuboidBucket.createBucket() else: s3backend = S3_FALSE unituser = User.objects.get(username=user) ds = NDDataset(Dataset ( dataset_name=dataset_name, user=unituser, ximagesize=ximagesize, yimagesize=yimagesize, zimagesize=zimagesize, xoffset=0, yoffset=0, zoffset=1, xvoxelres=xvoxelres, yvoxelres=yvoxelres, zvoxelres=zvoxelres, scalingoption=scalingoption, scalinglevels=scalinglevels, public=PUBLIC_TRUE, dataset_description="Unit test" ) ) ds.create() # make the project entry pr = NDProject(Project(project_name=project_name, project_description='Unit test', user=unituser, dataset=ds._ds, nd_version=nd_version, host='localhost', kvengine=KV_ENGINE, kvserver=KV_SERVER, s3backend=s3backend)) pr.create() # create a token tk = NDToken(Token (token_name = token_name, user = unituser, token_description = 'Unit test token', project_id = pr.project_name, public = public)) tk.create() # get the correct object for the kvengine # pd = NDProjectsDB.getProjDB(NDProjectpr) # create the database # pd.newNDProject() try: for channel_name in channel_list: ch = NDChannel(Channel (channel_name=channel_name, channel_type=channel_type, channel_datatype=channel_datatype, channel_description='Unit test channel', project_id=pr.project_name, readonly=readonly, propagate=propagate, resolution=base_resolution, exceptions=1, starttime=time[0], endtime=time[1] ,startwindow=window[0], endwindow=window[1], default=default)) # create a channel ch.create() # create the channel table # pd.newNDChannel(ch.channel_name) except Exception, e: print(e) raise e
def deleteTestDB ( project_name, token_name='unittest' ): try: # get the objects tk = NDToken.fromName(token_name) tk.delete() pr = NDProject.fromName(project_name) ds = pr.datasetcfg # tk = Token.objects.get(token_name=token_name) # pr = Project.objects.get(project_name=project_name) # ds = Dataset.objects.get(dataset_name=pr.dataset_id) # get the channel list # channel_list = Channel.objects.filter(project_id=pr) # get the correct object for the kvengine # pd = NDProjectsDB.getProjDB(pr) for ch in pr.projectChannels(): ch.delete() # delete the channel table # pd.deleteNDChannel(ch.channel_name) # delete the channel # ch.delete() # delete the project database # pd.deleteNDProject() # delete the objects pr.delete() ds.delete() # delete s3 bucket and dynamo table if KV_ENGINE == REDIS: CuboidIndexDB.deleteTable() CuboidBucket.deleteBucket() except Exception, e: print(e) raise e
class S3Cuboid(object): def __init__(self, token_name, host_name=HOST_NAME): # configuring the logger based on the dataset we are uploading self.logger = logging.getLogger(token_name) self.logger.setLevel(logging.INFO) fh = logging.FileHandler('{}_upload.log'.format(token_name)) self.logger.addHandler(fh) self.info_interface = InfoInterface(host_name, token_name) self.project_name = self.info_interface.project_name self.cuboidindex_db = CuboidIndexDB(self.project_name) self.cuboid_bucket = CuboidBucket(self.project_name) def upload(self, file_name, channel_name, resolution, x_index, y_index, z_index, dimensions=[1, 64, 512,512], time_index=0, neariso=False): """Upload a 4D supercuboid directly to dynamo and s3""" cuboid_data = np.fromfile(file_name, dtype=self.info_interface.get_channel_datatype(channel_name)) cuboid_data = cuboid_data.reshape(dimensions) super_zidx = XYZMorton([x_index, y_index, z_index]) self.logger.info("Inserting cube {},{},{}".format(x_index, y_index, z_index)) self.cuboidindex_db.putItem(channel_name, resolution, x_index, y_index, z_index, time_index, neariso=neariso) self.cuboid_bucket.putObject(channel_name, resolution, super_zidx, time_index, blosc.pack_array(cuboid_data), neariso=neariso)
def deleteTestDB(project_name, token_name='unittest'): try: # get the objects tk = NDToken.fromName(token_name) tk.delete() pr = NDProject.fromName(project_name) ds = pr.datasetcfg # tk = Token.objects.get(token_name=token_name) # pr = Project.objects.get(project_name=project_name) # ds = Dataset.objects.get(dataset_name=pr.dataset_id) # get the channel list # channel_list = Channel.objects.filter(project_id=pr) # get the correct object for the kvengine # pd = NDProjectsDB.getProjDB(pr) for ch in pr.projectChannels(): ch.delete() # delete the channel table # pd.deleteNDChannel(ch.channel_name) # delete the channel # ch.delete() # delete the project database # pd.deleteNDProject() # delete the objects pr.delete() ds.delete() # delete s3 bucket and dynamo table if KV_ENGINE == REDIS: CuboidIndexDB.deleteTable() CuboidBucket.deleteBucket() except Exception, e: print(e) raise e
def __init__(self, token_name, host_name=HOST_NAME): """Create the bucket and intialize values""" # configuring the logger based on the dataset we are uploading self.logger = logging.getLogger(token_name) self.logger.setLevel(logging.INFO) fh = logging.FileHandler('{}.log'.format(token_name)) self.logger.addHandler(fh) # setting up the project metadata self.info_interface = InfoInterface(host_name, token_name) # creating the resource interface to the remote server # self.resource_interface = ResourceInterface(self.info_interface.dataset_name, self.info_interface.project_name, host_name, logger=self.logger) # self.proj = self.resource_interface.getProject() # create the s3 I/O and index objects self.cuboidindex_db = CuboidIndexDB(self.info_interface.project_name) self.cuboid_bucket = CuboidBucket(self.info_interface.project_name)
class S3ProjectDB: """Database for the projects""" def __init__(self, pr): """Create the database connection""" self.pr = pr # create connections for cuboid bucket and cuboid dyanmo table self.cuboid_bucket = CuboidBucket(self.pr.project_name, endpoint_url=ndingest_settings.S3_ENDPOINT) self.cuboidindex_db = CuboidIndexDB(self.pr.project_name, endpoint_url=ndingest_settings.DYNAMO_ENDPOINT) def __del__(self): """Close the database connection""" self.close() def close (self): """Close the database connection""" pass def newNDProject(self): """Create the database for a project.""" pass def newNDChannel(self, channel_name): """Create the tables for a channel.""" pass def deleteNDProject(self): """Delete a project in s3 and dyanmo""" try: for item in self.cuboidindex_db.queryProjectItems(): self.cuboid_bucket.deleteObject(item['supercuboid_key']) self.cuboidindex_db.deleteItem(item['supercuboid_key']) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'ResourceNotFoundException': logger.warning("Resource was not accessible {}".format(e)) pass else: raise e except Exception as e: logger.error("Error in deleting S3 project {}. {}".format(self.pr.project_name, e)) raise NDWSError("Error in deleting S3 project {}. {}".format(self.pr.project_name, e)) def deleteNDChannel(self, channel_name): """Delete a channel in s3 and dynamo""" try: for item in self.cuboidindex_db.queryChannelItems(channel_name): self.cuboid_bucket.deleteObject(item['supercuboid_key']) self.cuboidindex_db.deleteItem(item['supercuboid_key']) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'ResourceNotFoundException': logger.warning("Resource was not accessible {}".format(e)) pass else: raise e except Exception as e: logger.error("Error in deleting S3 channel {}. {}".format(channel_name, e)) raise NDWSError("Error in deleting S3 channel {}. {}".format(channel_name, e)) def deleteNDResolution(self, channel_name, resolution): """Delete the resolution in s3 and dynamo""" try: for item in self.cuboidindex_db.queryResolutionItems(channel_name, resolution): self.cuboid_bucket.deleteObject(item['supercuboid_key']) self.cuboidindex_db.deleteItem(item['supercuboid_key']) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'ResourceNotFoundException': logger.warning("Resource was not accessible {}".format(e)) pass else: raise e except Exception as e: logger.error("Error in deleting S3 channel resolution {},{}. {}".format(channel_name, resolution, e)) raise NDWSError("Error in deleting S3 channel resolution {},{}. {}".format(channel_name, resolution, e))
class AwsInterface: def __init__(self, token_name, host_name=HOST_NAME): """Create the bucket and intialize values""" # configuring the logger based on the dataset we are uploading self.logger = logging.getLogger(token_name) self.logger.setLevel(logging.INFO) fh = logging.FileHandler('{}.log'.format(token_name)) self.logger.addHandler(fh) # setting up the project metadata self.info_interface = InfoInterface(host_name, token_name) # creating the resource interface to the remote server # self.resource_interface = ResourceInterface(self.info_interface.dataset_name, self.info_interface.project_name, host_name, logger=self.logger) # self.proj = self.resource_interface.getProject() # create the s3 I/O and index objects self.cuboidindex_db = CuboidIndexDB(self.info_interface.project_name) self.cuboid_bucket = CuboidBucket(self.info_interface.project_name) # def setupNewProject(self): # """Setup a new project if it does not exist""" # self.resource_interface.createDataset() # self.resource_interface.createProject() # self.resource_interface.createToken() def uploadExistingProject(self, channel_name, resolution, start_values, neariso=False): """Upload an existing project to S3""" self.setupNewProject() db = SpatialDB(self.proj) # checking for channels if channel_name is None: channel_list = None else: channel_list = [channel_name] # iterating over channels in a project for ch in self.proj.projectChannels(channel_list): # creating the channel resource self.resource_interface.createChannel(ch.channel_name) # ingest 1 or more resolutions based on user input if resolution is None: start_res = self.proj.datasetcfg.scalinglevels stop_res = ch.resolution - 1 else: start_res = resolution stop_res = resolution - 1 # iterating over resolution for cur_res in range(start_res, stop_res, -1): # get the source database sizes [image_size, time_range] = self.proj.datasetcfg.dataset_dim(cur_res) [xcubedim, ycubedim, zcubedim ] = cubedim = self.proj.datasetcfg.get_cubedim(cur_res) offset = self.proj.datasetcfg.get_offset(cur_res) [xsupercubedim, ysupercubedim, zsupercubedim ] = supercubedim = self.proj.datasetcfg.get_supercubedim( cur_res) # set the limits for iteration on the number of cubes in each dimension xlimit = (image_size[0] - 1) / (xsupercubedim) + 1 ylimit = (image_size[1] - 1) / (ysupercubedim) + 1 zlimit = (image_size[2] - 1) / (zsupercubedim) + 1 # [xlimit, ylimit, zlimit] = limit = self.proj.datasetcfg.get_supercube_limit(cur_res) [x_start, y_start, z_start] = map(div, start_values, supercubedim) for z in range(z_start, zlimit, 1): for y in range(y_start, ylimit, 1): for x in range(x_start, xlimit, 1): try: # cutout the data at the current resolution data = db.cutout(ch, [ x * xsupercubedim, y * ysupercubedim, z * zsupercubedim ], [ xsupercubedim, ysupercubedim, zsupercubedim ], cur_res).data # generate the morton index morton_index = XYZMorton([x, y, z]) self.logger.info("[{},{},{}] at res {}".format( x * xsupercubedim, y * ysupercubedim, z * zsupercubedim, cur_res)) # updating the index # self.cuboidindex_db.putItem(ch.channel_name, cur_res, x, y, z, ch.time_range[0]) # inserting the cube self.s3_io.putCube(ch, ch.time_stamp[0], morton_index, cur_res, blosc.pack_array(data), neariso=neariso) except Exception as e: # checkpoint the ingest self.logger.error(e) self.checkpoint_ingest(ch.channel_name, cur_res, x, y, z, e) raise e def uploadNewProject(self, config_file, start_values, neariso=False): """Upload a new project""" # loading the config file and assdociated params and processors config = Configuration() config.load(json.loads(open(config_file, 'rt').read())) config.load_plugins() path_processor = config.path_processor_class path_processor.setup(config.get_path_processor_params()) tile_processor = config.tile_processor_class tile_processor.setup(config.get_tile_processor_params()) tile_params = config.get_tile_processor_params() path_params = config.get_path_processor_params() # creating the channel object from resource service channel_name = config.config_data['database']['channel'] channel_datatype = self.info_interface.get_channel_datatype( channel_name) cur_res = tile_params['ingest_job']['resolution'] # loading all the parameters for image-sizes, tile-sizes, and iteration limits [xsupercubedim, ysupercubedim, zsupercubedim] = supercubedim = SUPER_CUBOID_SIZE [x_start, x_end] = tile_params['ingest_job']['extent']['x'] [y_start, y_end] = tile_params['ingest_job']['extent']['y'] [z_start, z_end] = tile_params['ingest_job']['extent']['z'] [t_start, t_end] = tile_params['ingest_job']['extent']['t'] x_tilesz = tile_params['ingest_job']['tile_size']['x'] y_tilesz = tile_params['ingest_job']['tile_size']['y'] z_tilesz = tile_params['ingest_job']['tile_size']['z'] t_tilesz = tile_params['ingest_job']['tile_size']['t'] x_limit = (x_end - 1) / (x_tilesz) + 1 y_limit = (y_end - 1) / (y_tilesz) + 1 z_limit = (z_end - 1) / (z_tilesz) + 1 t_limit = (t_end - 1) / (t_tilesz) + 1 if start_values != [0, 0, 0]: [x_start, y_start, z_start] = map(div, start_values, [x_tilesz, y_tilesz, z_tilesz]) # iterate over t,z,y,x to ingest the data for t in range(t_start, t_limit, 1): for z in range(z_start, z_limit, zsupercubedim): for y in range(y_start, y_limit, 1): for x in range(x_start, x_limit, 1): data = np.zeros([zsupercubedim, y_tilesz, x_tilesz], dtype=ND_dtypetonp[channel_datatype]) for b in range(0, zsupercubedim, 1): if z + b > z_end - 1: break # generate file name file_name = path_processor.process(x, y, z + b, t) # read the file, handle expection if the file is missing try: tile_handle = tile_processor.process( file_name, x, y, z + b, t) tile_handle.seek(0) data[b, :, :] = np.asarray( Image.open(tile_handle)) except IOError as e: pass # print "missing file", file_name # iterate over the tile if it is larger then supercuboid size for y_index in range(0, y_tilesz / ysupercubedim): for x_index in range(0, x_tilesz / xsupercubedim): # calculate the morton index insert_data = data[:, y_index * ysupercubedim:(y_index + 1) * ysupercubedim, x_index * xsupercubedim:(x_index + 1) * xsupercubedim] if np.any(insert_data): morton_index = XYZMorton([ x_index + (x * x_tilesz / xsupercubedim), y_index + (y * y_tilesz / ysupercubedim), z / zsupercubedim ]) [s3_x, s3_y, s3_z] = MortonXYZ(morton_index) print "Morton Index {}".format( morton_index) self.logger.info("[{},{},{}]".format( (x_index + x) * x_tilesz, (y_index + y) * y_tilesz, z)) self.cuboidindex_db.putItem( channel_name, cur_res, s3_x, s3_y, s3_z, t, neariso=neariso) self.cuboid_bucket.putObject( channel_name, cur_res, morton_index, t, blosc.pack_array(insert_data), neariso=neariso) # self.s3_io.putCube(ch, t, morton_index, cur_res, blosc.pack_array(insert_data), update=False, neariso=False) def checkpoint_ingest(self, channel_name, resolution, x, y, z, e, time=0): """Checkpoint the progress to file""" with closing(open('checkpoint_ingest.csv', 'wb')) as csv_file: field_names = [ 'project_name', 'channel_name', 'resolution', 'x', 'y', 'z', 'time', 'exception' ] csv_writer = csv.DictWriter(csv_file, delimiter=',', fieldnames=field_names) csv_writer.writeheader() csv_writer.writerow({ 'project_name': self.proj.project_name, 'channel_name': channel_name, 'resolution': resolution, 'x': x, 'y': y, 'z': z, 'time': time, 'exception': e.message }) def load_checkpoint(self): """Load from a checkpoint file""" return NotImplemented
def __init__(self, dataset_name, project_name, host_name=HOST_NAME): # self.resource_interface = ResourceInterface(dataset_name, project_name, host_name) self.cuboid_bucket = CuboidBucket(project_name) self.cuboidindex_db = CuboidIndexDB(project_name)
from nduser.models import Project from nduser.models import Dataset from nduser.models import Token from nduser.models import Channel from ndproj.nddataset import NDDataset from ndproj.ndproject import NDProject from ndproj.ndchannel import NDChannel from ndproj.ndtoken import NDToken from ndlib.ndtype import * from test_settings import * if KV_ENGINE == REDIS: from ndingest.nddynamo.cuboidindexdb import CuboidIndexDB from ndingest.ndbucket.cuboidbucket import CuboidBucket try: CuboidIndexDB.deleteTable() CuboidBucket.deleteBucket() except Exception as e: pass def createTestDB ( project_name, channel_list=['unit_anno'], channel_type=ANNOTATION, channel_datatype=UINT32, public=PUBLIC_TRUE, ximagesize=10000, yimagesize=10000, zimagesize=1000, xvoxelres=4.0, yvoxelres=4.0, zvoxelres=3.0, scalingoption=ZSLICES, scalinglevels=5, readonly=READONLY_FALSE, propagate=NOT_PROPAGATED, window=[0,0], time=[0,15], default=False, nd_version=ND_VERSION, token_name='unittest', user='******', dataset_name="unittest", base_resolution=0): """Create a unit test data base on the specified sit and name""" # setting s3backend to true if Redis and creating s3 bucket and dynamo table if KV_ENGINE == REDIS: s3backend = S3_TRUE CuboidIndexDB.createTable() CuboidBucket.createBucket() else: s3backend = S3_FALSE unituser = User.objects.get(username=user)
class AwsInterface: def __init__(self, token_name, host_name=HOST_NAME): """Create the bucket and intialize values""" # configuring the logger based on the dataset we are uploading self.logger = logging.getLogger(token_name) self.logger.setLevel(logging.INFO) fh = logging.FileHandler('{}.log'.format(token_name)) self.logger.addHandler(fh) # setting up the project metadata self.info_interface = InfoInterface(host_name, token_name) # creating the resource interface to the remote server # self.resource_interface = ResourceInterface(self.info_interface.dataset_name, self.info_interface.project_name, host_name, logger=self.logger) # self.proj = self.resource_interface.getProject() # create the s3 I/O and index objects self.cuboidindex_db = CuboidIndexDB(self.info_interface.project_name) self.cuboid_bucket = CuboidBucket(self.info_interface.project_name) # def setupNewProject(self): # """Setup a new project if it does not exist""" # self.resource_interface.createDataset() # self.resource_interface.createProject() # self.resource_interface.createToken() def uploadExistingProject(self, channel_name, resolution, start_values, neariso=False): """Upload an existing project to S3""" self.setupNewProject() db = SpatialDB(self.proj) # checking for channels if channel_name is None: channel_list = None else: channel_list = [channel_name] # iterating over channels in a project for ch in self.proj.projectChannels(channel_list): # creating the channel resource self.resource_interface.createChannel(ch.channel_name) # ingest 1 or more resolutions based on user input if resolution is None: start_res = self.proj.datasetcfg.scalinglevels stop_res = ch.resolution - 1 else: start_res = resolution stop_res = resolution - 1 # iterating over resolution for cur_res in range(start_res, stop_res, -1): # get the source database sizes [image_size, time_range] = self.proj.datasetcfg.dataset_dim(cur_res) [xcubedim, ycubedim, zcubedim] = cubedim = self.proj.datasetcfg.get_cubedim(cur_res) offset = self.proj.datasetcfg.get_offset(cur_res) [xsupercubedim, ysupercubedim, zsupercubedim] = supercubedim = self.proj.datasetcfg.get_supercubedim(cur_res) # set the limits for iteration on the number of cubes in each dimension xlimit = (image_size[0]-1) / (xsupercubedim) + 1 ylimit = (image_size[1]-1) / (ysupercubedim) + 1 zlimit = (image_size[2]-1) / (zsupercubedim) + 1 # [xlimit, ylimit, zlimit] = limit = self.proj.datasetcfg.get_supercube_limit(cur_res) [x_start, y_start, z_start] = map(div, start_values, supercubedim) for z in range(z_start, zlimit, 1): for y in range(y_start, ylimit, 1): for x in range(x_start, xlimit, 1): try: # cutout the data at the current resolution data = db.cutout(ch, [x*xsupercubedim, y*ysupercubedim, z*zsupercubedim], [xsupercubedim, ysupercubedim, zsupercubedim], cur_res).data # generate the morton index morton_index = XYZMorton([x, y, z]) self.logger.info("[{},{},{}] at res {}".format(x*xsupercubedim, y*ysupercubedim, z*zsupercubedim, cur_res)) # updating the index # self.cuboidindex_db.putItem(ch.channel_name, cur_res, x, y, z, ch.time_range[0]) # inserting the cube self.s3_io.putCube(ch, ch.time_stamp[0], morton_index, cur_res, blosc.pack_array(data), neariso=neariso) except Exception as e: # checkpoint the ingest self.logger.error(e) self.checkpoint_ingest(ch.channel_name, cur_res, x, y, z, e) raise e def uploadNewProject(self, config_file, start_values, neariso=False): """Upload a new project""" # loading the config file and assdociated params and processors config = Configuration() config.load(json.loads(open(config_file, 'rt').read())) config.load_plugins() path_processor = config.path_processor_class path_processor.setup(config.get_path_processor_params()) tile_processor = config.tile_processor_class tile_processor.setup(config.get_tile_processor_params()) tile_params = config.get_tile_processor_params() path_params = config.get_path_processor_params() # creating the channel object from resource service channel_name = config.config_data['database']['channel'] channel_datatype = self.info_interface.get_channel_datatype(channel_name) cur_res = tile_params['ingest_job']['resolution'] # loading all the parameters for image-sizes, tile-sizes, and iteration limits [xsupercubedim, ysupercubedim, zsupercubedim] = supercubedim = SUPER_CUBOID_SIZE [x_start, x_end] = tile_params['ingest_job']['extent']['x'] [y_start, y_end] = tile_params['ingest_job']['extent']['y'] [z_start, z_end] = tile_params['ingest_job']['extent']['z'] [t_start, t_end] = tile_params['ingest_job']['extent']['t'] x_tilesz = tile_params['ingest_job']['tile_size']['x'] y_tilesz = tile_params['ingest_job']['tile_size']['y'] z_tilesz = tile_params['ingest_job']['tile_size']['z'] t_tilesz = tile_params['ingest_job']['tile_size']['t'] x_limit = (x_end-1) / (x_tilesz) + 1 y_limit = (y_end-1) / (y_tilesz) + 1 z_limit = (z_end-1) / (z_tilesz) + 1 t_limit = (t_end-1) / (t_tilesz) + 1 if start_values != [0, 0, 0]: [x_start, y_start, z_start] = map(div, start_values, [x_tilesz, y_tilesz, z_tilesz]) # iterate over t,z,y,x to ingest the data for t in range(t_start, t_limit, 1): for z in range(z_start, z_limit, zsupercubedim): for y in range(y_start, y_limit, 1): for x in range(x_start, x_limit, 1): data = np.zeros([zsupercubedim, y_tilesz, x_tilesz], dtype=ND_dtypetonp[channel_datatype]) for b in range(0, zsupercubedim, 1): if z + b > z_end - 1: break # generate file name file_name = path_processor.process(x, y, z+b, t) # read the file, handle expection if the file is missing try: tile_handle = tile_processor.process(file_name, x, y, z+b, t) tile_handle.seek(0) data[b,:,:] = np.asarray(Image.open(tile_handle)) except IOError as e: pass # print "missing file", file_name # iterate over the tile if it is larger then supercuboid size for y_index in range(0, y_tilesz/ysupercubedim): for x_index in range(0, x_tilesz/xsupercubedim): # calculate the morton index insert_data = data[:, y_index*ysupercubedim:(y_index+1)*ysupercubedim, x_index*xsupercubedim:(x_index+1)*xsupercubedim] if np.any(insert_data): morton_index = XYZMorton([x_index+(x*x_tilesz/xsupercubedim), y_index+(y*y_tilesz/ysupercubedim), z/zsupercubedim]) [s3_x, s3_y, s3_z] = MortonXYZ(morton_index) print "Morton Index {}".format(morton_index) self.logger.info("[{},{},{}]".format((x_index+x)*x_tilesz, (y_index+y)*y_tilesz, z)) self.cuboidindex_db.putItem(channel_name, cur_res, s3_x, s3_y, s3_z, t, neariso=neariso) self.cuboid_bucket.putObject(channel_name, cur_res, morton_index, t, blosc.pack_array(insert_data), neariso=neariso) # self.s3_io.putCube(ch, t, morton_index, cur_res, blosc.pack_array(insert_data), update=False, neariso=False) def checkpoint_ingest(self, channel_name, resolution, x, y, z, e, time=0): """Checkpoint the progress to file""" with closing(open('checkpoint_ingest.csv', 'wb')) as csv_file: field_names = ['project_name', 'channel_name', 'resolution', 'x', 'y', 'z', 'time', 'exception'] csv_writer = csv.DictWriter(csv_file, delimiter=',', fieldnames=field_names) csv_writer.writeheader() csv_writer.writerow({'project_name' : self.proj.project_name, 'channel_name' : channel_name, 'resolution' : resolution, 'x' : x, 'y' : y, 'z' : z, 'time' : time, 'exception' : e.message}) def load_checkpoint(self): """Load from a checkpoint file""" return NotImplemented
class S3ProjectDB: """Database for the projects""" def __init__(self, pr): """Create the database connection""" self.pr = pr # create connections for cuboid bucket and cuboid dyanmo table self.cuboid_bucket = CuboidBucket( self.pr.project_name, endpoint_url=ndingest_settings.S3_ENDPOINT) self.cuboidindex_db = CuboidIndexDB( self.pr.project_name, endpoint_url=ndingest_settings.DYNAMO_ENDPOINT) def __del__(self): """Close the database connection""" self.close() def close(self): """Close the database connection""" pass def newNDProject(self): """Create the database for a project.""" pass def newNDChannel(self, channel_name): """Create the tables for a channel.""" pass def deleteNDProject(self): """Delete a project in s3 and dyanmo""" try: for item in self.cuboidindex_db.queryProjectItems(): self.cuboid_bucket.deleteObject(item['supercuboid_key']) self.cuboidindex_db.deleteItem(item['supercuboid_key']) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'ResourceNotFoundException': logger.warning("Resource was not accessible {}".format(e)) pass else: raise e except Exception as e: logger.error("Error in deleting S3 project {}. {}".format( self.pr.project_name, e)) raise NDWSError("Error in deleting S3 project {}. {}".format( self.pr.project_name, e)) def deleteNDChannel(self, channel_name): """Delete a channel in s3 and dynamo""" try: for item in self.cuboidindex_db.queryChannelItems(channel_name): self.cuboid_bucket.deleteObject(item['supercuboid_key']) self.cuboidindex_db.deleteItem(item['supercuboid_key']) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'ResourceNotFoundException': logger.warning("Resource was not accessible {}".format(e)) pass else: raise e except Exception as e: logger.error("Error in deleting S3 channel {}. {}".format( channel_name, e)) raise NDWSError("Error in deleting S3 channel {}. {}".format( channel_name, e)) def deleteNDResolution(self, channel_name, resolution): """Delete the resolution in s3 and dynamo""" try: for item in self.cuboidindex_db.queryResolutionItems( channel_name, resolution): self.cuboid_bucket.deleteObject(item['supercuboid_key']) self.cuboidindex_db.deleteItem(item['supercuboid_key']) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'ResourceNotFoundException': logger.warning("Resource was not accessible {}".format(e)) pass else: raise e except Exception as e: logger.error( "Error in deleting S3 channel resolution {},{}. {}".format( channel_name, resolution, e)) raise NDWSError( "Error in deleting S3 channel resolution {},{}. {}".format( channel_name, resolution, e))
# Copyright 2014 NeuroData (http://neurodata.io) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys sys.path += [os.path.abspath('../django')] import ND.settings os.environ['DJANGO_SETTINGS_MODULE'] = 'ND.settings' from ndingest.nddynamo.cuboidindexdb import CuboidIndexDB from ndingest.ndbucket.cuboidbucket import CuboidBucket # setup cuboid indext table and bucket locally CuboidIndexDB.createTable() CuboidBucket.createBucket()
def __init__(self, pr): """Create the database connection""" self.pr = pr # create connections for cuboid bucket and cuboid dyanmo table self.cuboid_bucket = CuboidBucket(self.pr.project_name, endpoint_url=ndingest_settings.S3_ENDPOINT) self.cuboidindex_db = CuboidIndexDB(self.pr.project_name, endpoint_url=ndingest_settings.DYNAMO_ENDPOINT)
def createTestDB(project_name, channel_list=['unit_anno'], channel_type=ANNOTATION, channel_datatype=UINT32, public=PUBLIC_TRUE, ximagesize=10000, yimagesize=10000, zimagesize=1000, xvoxelres=4.0, yvoxelres=4.0, zvoxelres=3.0, scalingoption=ZSLICES, scalinglevels=5, readonly=READONLY_FALSE, propagate=NOT_PROPAGATED, window=[0, 0], time=[0, 15], default=False, nd_version=ND_VERSION, token_name='unittest', user='******', dataset_name="unittest", base_resolution=0): """Create a unit test data base on the specified sit and name""" # setting s3backend to true if Redis and creating s3 bucket and dynamo table if KV_ENGINE == REDIS: s3backend = S3_TRUE CuboidIndexDB.createTable() CuboidBucket.createBucket() else: s3backend = S3_FALSE unituser = User.objects.get(username=user) ds = NDDataset( Dataset(dataset_name=dataset_name, user=unituser, ximagesize=ximagesize, yimagesize=yimagesize, zimagesize=zimagesize, xoffset=0, yoffset=0, zoffset=1, xvoxelres=xvoxelres, yvoxelres=yvoxelres, zvoxelres=zvoxelres, scalingoption=scalingoption, scalinglevels=scalinglevels, public=PUBLIC_TRUE, dataset_description="Unit test")) ds.create() # make the project entry pr = NDProject( Project(project_name=project_name, project_description='Unit test', user=unituser, dataset=ds._ds, nd_version=nd_version, host='localhost', kvengine=KV_ENGINE, kvserver=KV_SERVER, s3backend=s3backend)) pr.create() # create a token tk = NDToken( Token(token_name=token_name, user=unituser, token_description='Unit test token', project_id=pr.project_name, public=public)) tk.create() # get the correct object for the kvengine # pd = NDProjectsDB.getProjDB(NDProjectpr) # create the database # pd.newNDProject() try: for channel_name in channel_list: ch = NDChannel( Channel(channel_name=channel_name, channel_type=channel_type, channel_datatype=channel_datatype, channel_description='Unit test channel', project_id=pr.project_name, readonly=readonly, propagate=propagate, resolution=base_resolution, exceptions=1, starttime=time[0], endtime=time[1], startwindow=window[0], endwindow=window[1], default=default)) # create a channel ch.create() # create the channel table # pd.newNDChannel(ch.channel_name) except Exception, e: print(e) raise e
from nduser.models import Project from nduser.models import Dataset from nduser.models import Token from nduser.models import Channel from ndproj.nddataset import NDDataset from ndproj.ndproject import NDProject from ndproj.ndchannel import NDChannel from ndproj.ndtoken import NDToken from ndlib.ndtype import * from test_settings import * if KV_ENGINE == REDIS: from ndingest.nddynamo.cuboidindexdb import CuboidIndexDB from ndingest.ndbucket.cuboidbucket import CuboidBucket try: CuboidIndexDB.deleteTable() CuboidBucket.deleteBucket() except Exception as e: pass def createTestDB(project_name, channel_list=['unit_anno'], channel_type=ANNOTATION, channel_datatype=UINT32, public=PUBLIC_TRUE, ximagesize=10000, yimagesize=10000, zimagesize=1000, xvoxelres=4.0, yvoxelres=4.0, zvoxelres=3.0,
class Test_IngestLambda: def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url="http://localhost:8000") CuboidIndexDB.createTable(endpoint_url="http://localhost:8000") except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url="http://localhost:8000") # create the tile bucket TileBucket.createBucket(endpoint_url="http://localhost:4567") self.tile_bucket = TileBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") self.tiles = [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] message_id = "testing" receipt_handle = "123456" # insert SUPER_CUBOID_SIZE tiles in the bucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_handle = cStringIO.StringIO() self.tile_bucket.putObject( tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, message_id, receipt_handle, ) # creating the cuboid bucket CuboidBucket.createBucket(endpoint_url="http://localhost:4567") self.cuboid_bucket = CuboidBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") # create the ingest queue IngestQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568") self.ingest_queue = IngestQueue(nd_proj, endpoint_url="http://localhost:4568") # send message to the ingest queue morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, morton_index) response = self.ingest_queue.sendMessage(supercuboid_key) # create the cleanup queue CleanupQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568") def teardown_class(self): """Teardown class parameters""" # cleanup tilebucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_key = self.tile_bucket.encodeObjectKey( nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, ) self.tile_bucket.deleteObject(tile_key) morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, self.tiles) self.cuboid_bucket.deleteObject(supercuboid_key) # delete created entities TileIndexDB.deleteTable(endpoint_url="http://localhost:8000") CuboidIndexDB.deleteTable(endpoint_url="http://localhost:8000") IngestQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") CleanupQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") TileBucket.deleteBucket(endpoint_url="http://localhost:4567") try: CuboidBucket.deleteBucket(endpoint_url="http://localhost:4567") except Exception as e: pass def test_Uploadevent(self): """Testing the event""" # creating an emulambda function func = emulambda.import_lambda("ingestlambda.lambda_handler") # creating an emulambda event event = emulambda.parse_event( open("../ndlambda/functions/ingest/ingest_event.json").read()) # calling the emulambda function to invoke a lambda emulambda.invoke_lambda(func, event, None, 0, None) # testing if the supercuboid was inserted in the bucket morton_index = XYZMorton(self.tiles) cuboid = self.cuboid_bucket.getObject(nd_proj.channel_name, nd_proj.resolution, morton_index) # testing if the message was removed from the ingest queue for message in self.ingest_queue.receiveMessage(): # KL TODO write the message id into the JSON event file directly print(message)
class Test_Cuboid_Bucket: def setup_class(self): """Setup Parameters""" if "S3_ENDPOINT" in dir(settings): self.endpoint_url = settings.S3_ENDPOINT else: self.endpoint_url = None CuboidBucket.createBucket(endpoint_url=self.endpoint_url) self.cuboid_bucket = CuboidBucket( nd_proj.project_name, endpoint_url=self.endpoint_url ) def teardown_class(self): """Teardown Parameters""" # Ensure bucket empty before deleting. for objs in self.cuboid_bucket.getAllObjects(): self.cuboid_bucket.deleteObject(objs.key) CuboidBucket.deleteBucket(endpoint_url=self.endpoint_url) @pytest.mark.skipif( settings.PROJECT_NAME == "Boss", reason="putObject() not supported by the Boss" ) def test_put_object(self): """Testing put object""" cube_data = blosc.pack_array(np.zeros(settings.SUPER_CUBOID_SIZE)) for morton_index in range(0, 10, 1): self.cuboid_bucket.putObject( nd_proj.channel_name, nd_proj.resolution, morton_index, cube_data ) for morton_index in range(0, 10, 1): supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, morton_index ) self.cuboid_bucket.deleteObject(supercuboid_key) def test_put_object_by_key(self): hashm = hashlib.md5() hashm.update(b"test_cuboidbucket_data") cube_data = blosc.pack_array(np.zeros(settings.SUPER_CUBOID_SIZE)) for morton_index in range(0, 10, 1): key = "{}&{}".format(hashm.hexdigest(), morton_index) self.cuboid_bucket.putObjectByKey(key, cube_data) for morton_index in range(0, 10, 1): key = "{}&{}".format(hashm.hexdigest(), morton_index) self.cuboid_bucket.deleteObject(key)