class TestSQSQueue(unittest.TestCase):
def setUp(self):
queue_name = 'chunkflow-test'
print('test sqs queue using queue of {}'.format(queue_name))
# use small fetch_wait_time_seconds to make the test faster
# we should use 20 seconds in production run to use long pull
# see more details in the class documentation.
self.queue = SQSQueue(queue_name,
wait_if_empty=None,
fetch_wait_time_seconds=1,
retry_times=3)
def test_send_and_receive_message_list(self):
print('start sending messages...')
message_list = []
for i in range(23):
message_list.append(str(i))
self.queue.send_message_list(message_list)
print('start receiving messages in iteration...')
i = 0
for receipt_handle, message in self.queue:
print('received message: {}'.format(i))
self.assertTrue(int(message) in range(23))
i += 1
self.queue.delete(receipt_handle)
def setUp(self):
queue_name = 'chunkflow-test'
print('test sqs queue using queue of {}'.format(queue_name))
# use small fetch_wait_time_seconds to make the test faster
# we should use 20 seconds in production run to use long pull
# see more details in the class documentation.
self.queue = SQSQueue(queue_name,
wait_if_empty=None,
fetch_wait_time_seconds=1)
def generate_tasks(layer_path, mip, start, overlap, chunk_size, grid_size,
queue_name):
"""Generate tasks."""
bboxes = create_bounding_boxes(chunk_size,
overlap=overlap,
layer_path=layer_path,
start=start,
mip=mip,
grid_size=grid_size,
verbose=state['verbose'])
if queue_name is not None:
queue = SQSQueue(queue_name)
queue.send_message_list(bboxes)
else:
for bbox in bboxes:
task = get_initial_task()
task['bbox'] = bbox
task['log']['bbox'] = bbox.to_filename()
yield task
def fetch_task(queue_name, visibility_timeout):
"""Fetch task from queue."""
# This operator is actually a generator,
# it replaces old tasks to a completely new tasks and loop over it!
queue = SQSQueue(queue_name, visibility_timeout=visibility_timeout)
for task_handle, bbox_str in queue:
print('get task: ', bbox_str)
bbox = Bbox.from_filename(bbox_str)
# record the task handle to delete after the processing
task = get_initial_task()
task['queue'] = queue
task['task_handle'] = task_handle
task['bbox'] = bbox
task['log']['bbox'] = bbox.to_filename()
yield task