Example #1
0
 def setup_class(self):
     """Setup Parameters"""
     if 'S3_ENDPOINT' in dir(settings):
         self.endpoint_url = settings.S3_ENDPOINT
     else:
         self.endpoint_url = None
     CuboidBucket.createBucket(endpoint_url=self.endpoint_url)
     self.cuboid_bucket = CuboidBucket(nd_proj.project_name,
                                       endpoint_url=self.endpoint_url)
Example #2
0
    def setup_class(self):
        """Setup class parameters"""

        # create the tile index table. skip if it exists
        try:
            TileIndexDB.createTable(endpoint_url="http://localhost:8000")
            CuboidIndexDB.createTable(endpoint_url="http://localhost:8000")
        except Exception as e:
            pass
        self.tileindex_db = TileIndexDB(nd_proj.project_name,
                                        endpoint_url="http://localhost:8000")

        # create the tile bucket
        TileBucket.createBucket(endpoint_url="http://localhost:4567")
        self.tile_bucket = TileBucket(nd_proj.project_name,
                                      endpoint_url="http://localhost:4567")
        self.tiles = [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0]

        message_id = "testing"
        receipt_handle = "123456"
        # insert SUPER_CUBOID_SIZE tiles in the bucket
        for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1):
            tile_handle = cStringIO.StringIO()
            self.tile_bucket.putObject(
                tile_handle,
                nd_proj.channel_name,
                nd_proj.resolution,
                self.x_tile,
                self.y_tile,
                z_index,
                message_id,
                receipt_handle,
            )

        # creating the cuboid bucket
        CuboidBucket.createBucket(endpoint_url="http://localhost:4567")
        self.cuboid_bucket = CuboidBucket(nd_proj.project_name,
                                          endpoint_url="http://localhost:4567")

        # create the ingest queue
        IngestQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568")
        self.ingest_queue = IngestQueue(nd_proj,
                                        endpoint_url="http://localhost:4568")

        # send message to the ingest queue
        morton_index = XYZMorton(self.tiles)
        supercuboid_key = self.cuboid_bucket.generateSupercuboidKey(
            nd_proj.channel_name, nd_proj.resolution, morton_index)
        response = self.ingest_queue.sendMessage(supercuboid_key)

        # create the cleanup queue
        CleanupQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568")
Example #3
0
def createTestDB ( project_name, channel_list=['unit_anno'], channel_type=ANNOTATION, channel_datatype=UINT32, public=PUBLIC_TRUE, ximagesize=10000, yimagesize=10000, zimagesize=1000, xvoxelres=4.0, yvoxelres=4.0, zvoxelres=3.0, scalingoption=ZSLICES, scalinglevels=5, readonly=READONLY_FALSE, propagate=NOT_PROPAGATED, window=[0,0], time=[0,15], default=False, nd_version=ND_VERSION, token_name='unittest', user='******', dataset_name="unittest", base_resolution=0):
  """Create a unit test data base on the specified sit and name"""
  
  # setting s3backend to true if Redis and creating s3 bucket and dynamo table
  if KV_ENGINE == REDIS:
    s3backend = S3_TRUE
    CuboidIndexDB.createTable()
    CuboidBucket.createBucket()    
  else:
    s3backend = S3_FALSE

  unituser = User.objects.get(username=user)

  ds = NDDataset(Dataset ( dataset_name=dataset_name, user=unituser, ximagesize=ximagesize, yimagesize=yimagesize, zimagesize=zimagesize,  xoffset=0, yoffset=0, zoffset=1, xvoxelres=xvoxelres, yvoxelres=yvoxelres, zvoxelres=zvoxelres, scalingoption=scalingoption, scalinglevels=scalinglevels, public=PUBLIC_TRUE, dataset_description="Unit test" ) )
  ds.create()

  # make the project entry
  pr = NDProject(Project(project_name=project_name, project_description='Unit test', user=unituser, dataset=ds._ds, nd_version=nd_version, host='localhost', kvengine=KV_ENGINE, kvserver=KV_SERVER, s3backend=s3backend))
  pr.create()

  # create a token
  tk = NDToken(Token (token_name = token_name, user = unituser, token_description = 'Unit test token', project_id = pr.project_name, public = public))
  tk.create()
  
  # get the correct object for the kvengine
  # pd = NDProjectsDB.getProjDB(NDProjectpr)
  # create the database
  # pd.newNDProject()

  try:
    for channel_name in channel_list:
      ch = NDChannel(Channel (channel_name=channel_name, channel_type=channel_type, channel_datatype=channel_datatype, channel_description='Unit test channel', project_id=pr.project_name, readonly=readonly, propagate=propagate, resolution=base_resolution, exceptions=1, starttime=time[0], endtime=time[1]  ,startwindow=window[0], endwindow=window[1], default=default))
      # create a channel
      ch.create()
      # create the channel table
      # pd.newNDChannel(ch.channel_name)
  except Exception, e:
      print(e)
      raise e
Example #4
0
# Copyright 2014 NeuroData (http://neurodata.io)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys

sys.path += [os.path.abspath('../django')]
import ND.settings

os.environ['DJANGO_SETTINGS_MODULE'] = 'ND.settings'
from ndingest.nddynamo.cuboidindexdb import CuboidIndexDB
from ndingest.ndbucket.cuboidbucket import CuboidBucket

# setup cuboid indext table and bucket locally
CuboidIndexDB.createTable()
CuboidBucket.createBucket()
Example #5
0
def createTestDB(project_name,
                 channel_list=['unit_anno'],
                 channel_type=ANNOTATION,
                 channel_datatype=UINT32,
                 public=PUBLIC_TRUE,
                 ximagesize=10000,
                 yimagesize=10000,
                 zimagesize=1000,
                 xvoxelres=4.0,
                 yvoxelres=4.0,
                 zvoxelres=3.0,
                 scalingoption=ZSLICES,
                 scalinglevels=5,
                 readonly=READONLY_FALSE,
                 propagate=NOT_PROPAGATED,
                 window=[0, 0],
                 time=[0, 15],
                 default=False,
                 nd_version=ND_VERSION,
                 token_name='unittest',
                 user='******',
                 dataset_name="unittest",
                 base_resolution=0):
    """Create a unit test data base on the specified sit and name"""

    # setting s3backend to true if Redis and creating s3 bucket and dynamo table
    if KV_ENGINE == REDIS:
        s3backend = S3_TRUE
        CuboidIndexDB.createTable()
        CuboidBucket.createBucket()
    else:
        s3backend = S3_FALSE

    unituser = User.objects.get(username=user)

    ds = NDDataset(
        Dataset(dataset_name=dataset_name,
                user=unituser,
                ximagesize=ximagesize,
                yimagesize=yimagesize,
                zimagesize=zimagesize,
                xoffset=0,
                yoffset=0,
                zoffset=1,
                xvoxelres=xvoxelres,
                yvoxelres=yvoxelres,
                zvoxelres=zvoxelres,
                scalingoption=scalingoption,
                scalinglevels=scalinglevels,
                public=PUBLIC_TRUE,
                dataset_description="Unit test"))
    ds.create()

    # make the project entry
    pr = NDProject(
        Project(project_name=project_name,
                project_description='Unit test',
                user=unituser,
                dataset=ds._ds,
                nd_version=nd_version,
                host='localhost',
                kvengine=KV_ENGINE,
                kvserver=KV_SERVER,
                s3backend=s3backend))
    pr.create()

    # create a token
    tk = NDToken(
        Token(token_name=token_name,
              user=unituser,
              token_description='Unit test token',
              project_id=pr.project_name,
              public=public))
    tk.create()

    # get the correct object for the kvengine
    # pd = NDProjectsDB.getProjDB(NDProjectpr)
    # create the database
    # pd.newNDProject()

    try:
        for channel_name in channel_list:
            ch = NDChannel(
                Channel(channel_name=channel_name,
                        channel_type=channel_type,
                        channel_datatype=channel_datatype,
                        channel_description='Unit test channel',
                        project_id=pr.project_name,
                        readonly=readonly,
                        propagate=propagate,
                        resolution=base_resolution,
                        exceptions=1,
                        starttime=time[0],
                        endtime=time[1],
                        startwindow=window[0],
                        endwindow=window[1],
                        default=default))
            # create a channel
            ch.create()
            # create the channel table
            # pd.newNDChannel(ch.channel_name)
    except Exception, e:
        print(e)
        raise e