# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
# from __future__ import absolute_import
from ndingest.settings.settings import Settings
settings = Settings.load()
import boto3
import botocore
from boto3.dynamodb.conditions import Key, Attr
import blosc
from ndctypelib import XYZMorton
from ndingest.util.util import Util
UtilClass = Util.load()
class CuboidIndexDB:
"""DynamoDB index of cuboids stored in S3.
This class is not used by the Boss. The Boss sets up its DynamoDB table via
CloudFormation.
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import botocore
import boto3
import redis
from ndproj.ndproject import NDProject
from ndproj.ndchannel import NDChannel
from webservices.ndwserror import NDWSError
from ndingest.nddynamo.cuboidindexdb import CuboidIndexDB
from ndingest.ndbucket.cuboidbucket import CuboidBucket
from ndingest.settings.settings import Settings
ndingest_settings = Settings.load()
import logging
logger=logging.getLogger("neurodata")
class S3ProjectDB:
"""Database for the projects"""
def __init__(self, pr):
"""Create the database connection"""
self.pr = pr
# create connections for cuboid bucket and cuboid dyanmo table
self.cuboid_bucket = CuboidBucket(self.pr.project_name, endpoint_url=ndingest_settings.S3_ENDPOINT)
self.cuboidindex_db = CuboidIndexDB(self.pr.project_name, endpoint_url=ndingest_settings.DYNAMO_ENDPOINT)
def __del__(self):
"""Close the database connection"""
from __future__ import absolute_import
import os
import sys
import json
from contextlib import closing
import argparse
import csv
import blosc
from operator import div
import numpy as np
from PIL import Image
sys.path.append(os.path.abspath('../django'))
import ND.settings
os.environ['DJANGO_SETTINGS_MODULE'] = 'ND.settings'
from ndingest.settings.settings import Settings
ndingest_settings = Settings.load()
import django
django.setup()
from django.forms.models import model_to_dict
from django.conf import settings
from ndlib.ndctypelib import *
from ndcube.cube import Cube
from ndlib.ndtype import *
from ndlib.restutil import *
from ndproj.nddataset import NDDataset
from ndproj.ndchannel import NDChannel
from ndproj.ndproject import NDProject
from ndproj.ndtoken import NDToken
from ndingest.nddynamo.cuboidindexdb import CuboidIndexDB
from ndingest.ndbucket.cuboidbucket import CuboidBucket
from scripts_helper import *
import blosc
import botocore
import django
django.setup()
from django.conf import settings
from spdb.ndcube.cube import Cube
from ndlib.ndtype import *
import webservices.ndwsrest as ndwsrest
from spdb.spatialdb import SpatialDB
from ndproj.ndprojdb import NDProjectsDB
from ndlib.ndctypelib import XYZMorton
from spdb.ndkvio.s3io import S3IO
from ndingest.nddynamo.cuboidindexdb import CuboidIndexDB
from webservices.ndwserror import NDWSError
from ndingest.settings.settings import Settings
ndsettings = Settings.load()
import logging
logger = logging.getLogger("neurodata")
class IngestData:
def __init__(self, token, channel, resolution, data_url, file_format, file_type):
self.token = token
self.channel = channel
self.resolution = resolution
self.data_url = data_url
self.path = settings.TEMP_INGEST_PATH
# identify which type of data this is
self.file_format = file_format
# set the file_type