def test_sendBatchMessages(sqs):
fake_data0 = {"foo": "bar"}
fake_data1 = {"john": "doe"}
jsonized0 = json.dumps(fake_data0)
jsonized1 = json.dumps(fake_data1)
md5_0 = hashlib.md5(jsonized0.encode("utf-8")).hexdigest()
md5_1 = hashlib.md5(jsonized1.encode("utf-8")).hexdigest()
proj = generate_proj()
from ndingest.ndqueue.uploadqueue import UploadQueue
UploadQueue.createQueue(proj)
upload_queue = UploadQueue(proj)
try:
response = upload_queue.sendBatchMessages([jsonized0, jsonized1], 0)
assert "Successful" in response
success_ids = []
for msg_result in response["Successful"]:
id = msg_result["Id"]
success_ids.append(id)
if id == "0":
assert md5_0 == msg_result["MD5OfMessageBody"]
elif id == "1":
assert md5_1 == msg_result["MD5OfMessageBody"]
assert "0" in success_ids
assert "1" in success_ids
finally:
for message_id, receipt_handle, _ in upload_queue.receiveMessage():
upload_queue.deleteMessage(message_id, receipt_handle)
def enqueue_msgs(fp):
"""Parse given messages and send to SQS queue.
Args:
fp (file-like-object): File-like-object containing a header and messages.
"""
read_header = False
msgs = []
upload_queue = None
lineNum = 0
for line in fp:
lineNum += 1
if not read_header:
header = json.loads(line)
if 'upload_queue_url' not in header:
raise KeyError('Expected upload_queue_url in header')
if 'ingest_queue_url' not in header:
raise KeyError('Expected ingest_queue_url in header')
if 'job_id' not in header:
raise KeyError('Expected job_id in header')
read_header = True
continue
try:
msgs.append(parse_line(header, line))
except:
print('Error parsing line {}: {}'.format(lineNum, line))
if len(msgs) == 1 and upload_queue is None:
# Instantiate the upload queue object.
asDict = json.loads(msgs[0])
boss_ingest_proj = BossIngestProj.fromTileKey(asDict['tile_key'])
boss_ingest_proj.job_id = header['job_id']
upload_queue = UploadQueue(boss_ingest_proj)
if len(msgs) >= MAX_BATCH_MSGS:
# Enqueue messages.
upload_queue.sendBatchMessages(msgs)
msgs = []
if len(msgs) > 0:
# Final enqueue messages of remaining messages.
upload_queue.sendBatchMessages(msgs)
def enqueue_msgs(fp):
"""Parse given messages and send to SQS queue.
Args:
fp (file-like-object): File-like-object containing a header and messages.
"""
read_header = False
msgs = []
upload_queue = None
lineNum = 0
for line in fp:
lineNum += 1
if not read_header:
header = json.loads(line)
if 'upload_queue_url' not in header:
raise KeyError('Expected upload_queue_url in header')
if 'ingest_queue_url' not in header:
raise KeyError('Expected ingest_queue_url in header')
if 'job_id' not in header:
raise KeyError('Expected job_id in header')
read_header = True
continue
try:
msgs.append(parse_line(header, line))
except:
print('Error parsing line {}: {}'.format(lineNum, line))
if len(msgs) == 1 and upload_queue is None:
# Instantiate the upload queue object.
asDict = json.loads(msgs[0])
boss_ingest_proj = BossIngestProj.fromTileKey(asDict['tile_key'])
boss_ingest_proj.job_id = header['job_id']
upload_queue = UploadQueue(boss_ingest_proj)
if len(msgs) >= MAX_BATCH_MSGS:
# Enqueue messages.
upload_queue.sendBatchMessages(msgs)
msgs = []
if len(msgs) > 0:
# Final enqueue messages of remaining messages.
upload_queue.sendBatchMessages(msgs)
def send_upload_message_batch(self, list_msg):
"""
Upload a batch of 10 messages to the upload queue. An error is raised if more than 10 messages are in the batch
Args:
list_msg: The list containing the messages to upload
Returns:
None
"""
queue = UploadQueue(self.nd_proj, endpoint_url=None)
status = queue.sendBatchMessages(list_msg)
return status