def test_sendBatchMessages(sqs): fake_data0 = {"foo": "bar"} fake_data1 = {"john": "doe"} jsonized0 = json.dumps(fake_data0) jsonized1 = json.dumps(fake_data1) md5_0 = hashlib.md5(jsonized0.encode("utf-8")).hexdigest() md5_1 = hashlib.md5(jsonized1.encode("utf-8")).hexdigest() proj = generate_proj() from ndingest.ndqueue.uploadqueue import UploadQueue UploadQueue.createQueue(proj) upload_queue = UploadQueue(proj) try: response = upload_queue.sendBatchMessages([jsonized0, jsonized1], 0) assert "Successful" in response success_ids = [] for msg_result in response["Successful"]: id = msg_result["Id"] success_ids.append(id) if id == "0": assert md5_0 == msg_result["MD5OfMessageBody"] elif id == "1": assert md5_1 == msg_result["MD5OfMessageBody"] assert "0" in success_ids assert "1" in success_ids finally: for message_id, receipt_handle, _ in upload_queue.receiveMessage(): upload_queue.deleteMessage(message_id, receipt_handle)
def enqueue_msgs(fp): """Parse given messages and send to SQS queue. Args: fp (file-like-object): File-like-object containing a header and messages. """ read_header = False msgs = [] upload_queue = None lineNum = 0 for line in fp: lineNum += 1 if not read_header: header = json.loads(line) if 'upload_queue_url' not in header: raise KeyError('Expected upload_queue_url in header') if 'ingest_queue_url' not in header: raise KeyError('Expected ingest_queue_url in header') if 'job_id' not in header: raise KeyError('Expected job_id in header') read_header = True continue try: msgs.append(parse_line(header, line)) except: print('Error parsing line {}: {}'.format(lineNum, line)) if len(msgs) == 1 and upload_queue is None: # Instantiate the upload queue object. asDict = json.loads(msgs[0]) boss_ingest_proj = BossIngestProj.fromTileKey(asDict['tile_key']) boss_ingest_proj.job_id = header['job_id'] upload_queue = UploadQueue(boss_ingest_proj) if len(msgs) >= MAX_BATCH_MSGS: # Enqueue messages. upload_queue.sendBatchMessages(msgs) msgs = [] if len(msgs) > 0: # Final enqueue messages of remaining messages. upload_queue.sendBatchMessages(msgs)
def enqueue_msgs(fp): """Parse given messages and send to SQS queue. Args: fp (file-like-object): File-like-object containing a header and messages. """ read_header = False msgs = [] upload_queue = None lineNum = 0 for line in fp: lineNum += 1 if not read_header: header = json.loads(line) if 'upload_queue_url' not in header: raise KeyError('Expected upload_queue_url in header') if 'ingest_queue_url' not in header: raise KeyError('Expected ingest_queue_url in header') if 'job_id' not in header: raise KeyError('Expected job_id in header') read_header = True continue try: msgs.append(parse_line(header, line)) except: print('Error parsing line {}: {}'.format(lineNum, line)) if len(msgs) == 1 and upload_queue is None: # Instantiate the upload queue object. asDict = json.loads(msgs[0]) boss_ingest_proj = BossIngestProj.fromTileKey(asDict['tile_key']) boss_ingest_proj.job_id = header['job_id'] upload_queue = UploadQueue(boss_ingest_proj) if len(msgs) >= MAX_BATCH_MSGS: # Enqueue messages. upload_queue.sendBatchMessages(msgs) msgs = [] if len(msgs) > 0: # Final enqueue messages of remaining messages. upload_queue.sendBatchMessages(msgs)
def send_upload_message_batch(self, list_msg): """ Upload a batch of 10 messages to the upload queue. An error is raised if more than 10 messages are in the batch Args: list_msg: The list containing the messages to upload Returns: None """ queue = UploadQueue(self.nd_proj, endpoint_url=None) status = queue.sendBatchMessages(list_msg) return status