def setUpClass(cls):
# Suppress warnings about Boto3's unclosed sockets.
warnings.simplefilter('ignore')
# Use ndingest in test mode.
os.environ['NDINGEST_TEST'] = '1'
cls.job_id = 125
cls.nd_proj = BossIngestProj('testCol', 'kasthuri11', 'image', 0,
cls.job_id)
UploadQueue.createQueue(cls.nd_proj)
cls.upload_queue = UploadQueue(cls.nd_proj)
def boss_util_fixtures(tile_bucket, sqs):
job_id = 123
nd_proj = BossIngestProj("testCol", "kasthuri11", "image", 0, job_id)
from ndingest.ndqueue.uploadqueue import UploadQueue
UploadQueue.createQueue(nd_proj)
upload_queue = UploadQueue(nd_proj)
from ndingest.ndqueue.tileindexqueue import TileIndexQueue
TileIndexQueue.createQueue(nd_proj)
tile_index_queue = TileIndexQueue(nd_proj)
def get_test_data():
return (nd_proj, upload_queue, tile_index_queue, tile_bucket)
yield get_test_data
UploadQueue.deleteQueue(nd_proj)
TileIndexQueue.deleteQueue(nd_proj)
def setUpClass(cls):
# Silence warnings about open boto3 sessions.
warnings.filterwarnings('ignore')
cls.job_id = 123
cls.nd_proj = BossIngestProj('testCol', 'kasthuri11', 'image', 0,
cls.job_id)
TileBucket.createBucket()
cls.tile_bucket = TileBucket(cls.nd_proj.project_name)
warnings.simplefilter('ignore')
#with open('/Users/manavpj1/repos/boss/django/bossingest/test/boss_tile_index.json') as fp:
# schema = json.load(fp)
#BossTileIndexDB.createTable(schema, endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
cls.tileindex_db = BossTileIndexDB(
cls.nd_proj.project_name,
endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
def populate_upload_queue(args):
"""Populate the ingest upload SQS Queue with tile information
Note: This activity will clear the upload queue of any existing
messages
Args:
args: {
'job_id': '',
'upload_queue': ARN,
'ingest_queue': ARN,
'collection_name': '',
'experiment_name': '',
'channel_name': '',
'resolution': 0,
'project_info': [col_id, exp_id, ch_id],
't_start': 0,
't_stop': 0,
't_tile_size': 0,
'x_start': 0,
'x_stop': 0,
'x_tile_size': 0,
'y_start': 0,
'y_stop': 0
'y_tile_size': 0,
'z_start': 0,
'z_stop': 0
'z_tile_size': 16,
}
Returns:
{'arn': Upload queue ARN,
'count': Number of messages put into the queue}
"""
log.debug("Starting to populate upload queue")
# DP NOTE: Could transform create_messages into a generator
# and yield after each sendBatchMessages
clear_queue(args['upload_queue'])
proj = BossIngestProj(args['collection_name'],
args['experiment_name'],
args['channel_name'],
args['resolution'],
args['job_id'])
queue = UploadQueue(proj)
msgs = create_messages(args)
sent = 0
# Implement a custom queue.sendBatchMessages so that more than 10
# messages can be sent at once (hard limit in sendBatchMessages)
while True:
batch = []
for i in range(SQS_BATCH_SIZE):
try:
batch.append({
'Id': str(i),
'MessageBody': next(msgs),
'DelaySeconds': 0
})
except StopIteration:
break
if len(batch) == 0:
break
retry = 3
while retry > 0:
resp = queue.queue.send_messages(Entries=batch)
sent += len(resp['Successful'])
if 'Failed' in resp and len(resp['Failed']) > 0:
log.debug("Batch failed to enqueue messages")
log.debug("Retries left: {}".format(retry))
log.debug("Boto3 send_messages response: {}".format(resp))
time.sleep(SQS_RETRY_TIMEOUT)
ids = [f['Id'] for f in resp['Failed']]
batch = [b for b in batch if b['Id'] in ids]
retry -= 1
if retry == 0:
log.debug("Exhausted retry count, stopping")
raise FailedToSendMessages(batch) # SFN will relaunch the activity
continue
else:
break
return {
'arn': args['upload_queue'],
'count': sent,
}
def test_project_name(self):
"""Project name should be collection name & experiment name."""
prj = BossIngestProj('col1', 'exp1', 'chanA', '0', '123')
expected = 'col1&exp1'
self.assertEqual(expected, prj.project_name)
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import absolute_import
import sys
sys.path.append('..')
from ndingest.settings.settings import Settings
settings = Settings.load()
from ndingest.nddynamo.boss_tileindexdb import BossTileIndexDB
from ndingest.ndingestproj.bossingestproj import BossIngestProj
job_id = '123'
nd_proj = BossIngestProj('testCol', 'kasthuri11', 'image', 0, job_id)
import json
import six
import unittest
import warnings
import time
import botocore
class Test_BossTileIndexDB(unittest.TestCase):
"""
Note that the chunk keys used, for testing, do not have real hash keys.
The rest of the chunk key is valid.
"""
def setUp(self):
# Suppress ResourceWarning messages about unclosed connections.
Note that these tests require a local DynamoDB to run against.
"""
from __future__ import print_function
from __future__ import absolute_import
import sys
sys.path.append("..")
from ndingest.settings.settings import Settings
settings = Settings.load()
from ndingest.nddynamo.boss_tileindexdb import BossTileIndexDB, TILE_UPLOADED_MAP_KEY
from ndingest.ndingestproj.bossingestproj import BossIngestProj
job_id = "123"
nd_proj = BossIngestProj("testCol", "kasthuri11", "image", 0, job_id)
import json
import six
import unittest
import warnings
import time
import botocore
class Test_BossTileIndexDB(unittest.TestCase):
"""
Note that the chunk keys used, for testing, do not have real hash keys.
The rest of the chunk key is valid.
"""
def setUp(self):
# Suppress ResourceWarning messages about unclosed connections.
def test_project_name(self):
"""Project name should be collection name & experiment name."""
prj = BossIngestProj("col1", "exp1", "chanA", "0", "123")
expected = "col1&exp1"
self.assertEqual(expected, prj.project_name)