def test_master_replaces_worker_processes(): """ Test managing process replaces worker processes """ # Setup SQS Queue conn = boto3.client('sqs', region_name='us-east-1') conn.create_queue(QueueName="tester") # Setup Manager manager = ManagerWorker( queue_prefixes=["tester"], worker_concurrency=1, interval=1, batchsize=10, ) manager.start() # Get Worker PID pid = manager.worker_children[0].pid # Kill Worker and wait to replace manager.worker_children[0].shutdown() time.sleep(0.1) manager.replace_workers() # Check Replacement manager.worker_children[0].pid.shouldnt.equal(pid) # Cleanup manager.stop()
def test_master_replaces_worker_processes(): """ Test managing process replaces worker processes """ # Setup SQS Queue conn = boto.connect_sqs() conn.create_queue("tester") # Setup Manager manager = ManagerWorker(queue_prefixes=["tester"], worker_concurrency=1) manager.start() # Get Worker PID pid = manager.worker_children[0].pid # Kill Worker and wait to replace manager.worker_children[0].shutdown() time.sleep(0.1) manager.replace_workers() # Check Replacement manager.worker_children[0].pid.shouldnt.equal(pid) # Cleanup manager.stop()
def test_manager_start_and_stop(): """ Test managing process can start and stop child processes """ conn = boto3.client('sqs', region_name='us-east-1') conn.create_queue(QueueName="email") manager = ManagerWorker(queue_prefixes=['email'], worker_concurrency=2, interval=1, batchsize=10) len(manager.worker_children).should.equal(2) manager.worker_children[0].is_alive().should.equal(False) manager.worker_children[1].is_alive().should.equal(False) manager.start() manager.worker_children[0].is_alive().should.equal(True) manager.worker_children[1].is_alive().should.equal(True) manager.stop() manager.worker_children[0].is_alive().should.equal(False) manager.worker_children[1].is_alive().should.equal(False)
def test_master_replaces_worker_processes(): """ Test managing process replaces worker processes """ # Setup SQS Queue conn = boto3.client('sqs', region_name='us-east-1') conn.create_queue(QueueName="tester") # Setup Manager manager = ManagerWorker(queue_prefixes=["tester"], worker_concurrency=1, interval=1, batchsize=10) manager.start() # Get Worker PID pid = manager.worker_children[0].pid # Kill Worker and wait to replace manager.worker_children[0].shutdown() time.sleep(0.1) manager.replace_workers() # Check Replacement manager.worker_children[0].pid.shouldnt.equal(pid) # Cleanup manager.stop()
def test_master_counts_processes(): """ Test managing process counts child processes """ # Setup Logging logger = logging.getLogger("pyqs") del logger.handlers[:] logger.handlers.append(MockLoggingHandler()) # Setup SQS Queue conn = boto3.client('sqs', region_name='us-east-1') conn.create_queue(QueueName="tester") # Setup Manager manager = ManagerWorker(["tester"], 2, 1, 10) manager.start() # Check Workers manager.process_counts() # Cleanup manager.stop() # Check messages msg1 = "Reader Processes: 1" logger.handlers[0].messages['debug'][-2].lower().should.contain( msg1.lower()) msg2 = "Worker Processes: 2" logger.handlers[0].messages['debug'][-1].lower().should.contain( msg2.lower())
def test_manager_picks_up_new_queues(): """ Test that the manager will recognize new SQS queues have been added """ # Setup SQS Queue conn = boto3.client('sqs', region_name='us-east-1') # Setup Manager manager = ManagerWorker( queue_prefixes=["tester"], worker_concurrency=1, interval=1, batchsize=10, ) manager.start() # No queues found len(manager.reader_children).should.equal(0) # Create the queue conn.create_queue(QueueName="tester") manager.check_for_new_queues() # The manager should have seen the new queue was created and add a reader len(manager.reader_children).should.equal(1) manager.reader_children[0].queue_url.should.equal( "https://queue.amazonaws.com/123456789012/tester") # Cleanup manager.stop()
def test_master_counts_processes(): """ Test managing process counts child processes """ # Setup Logging logger = logging.getLogger("pyqs") del logger.handlers[:] logger.handlers.append(MockLoggingHandler()) # Setup SQS Queue conn = boto.connect_sqs() conn.create_queue("tester") # Setup Manager manager = ManagerWorker(["tester"], 2) manager.start() # Check Workers manager.process_counts() # Cleanup manager.stop() # Check messages msg1 = "Reader Processes: 1" logger.handlers[0].messages['debug'][-2].lower().should.contain(msg1.lower()) msg2 = "Worker Processes: 2" logger.handlers[0].messages['debug'][-1].lower().should.contain(msg2.lower())
def test_manager_start_and_stop(): """ Test managing process can start and stop child processes """ conn = boto.connect_sqs() conn.create_queue("email") manager = ManagerWorker(queue_prefixes=['email'], worker_concurrency=2, interval=1, batchsize=10) len(manager.worker_children).should.equal(2) manager.worker_children[0].is_alive().should.equal(False) manager.worker_children[1].is_alive().should.equal(False) manager.start() manager.worker_children[0].is_alive().should.equal(True) manager.worker_children[1].is_alive().should.equal(True) manager.stop() manager.worker_children[0].is_alive().should.equal(False) manager.worker_children[1].is_alive().should.equal(False)
def test_manager_start_and_stop(): """ Test managing process can start and stop child processes """ conn = boto3.client('sqs', region_name='us-east-1') conn.create_queue(QueueName="email") manager = ManagerWorker( queue_prefixes=['email'], worker_concurrency=2, interval=1, batchsize=10, ) len(manager.worker_children).should.equal(2) manager.worker_children[0].is_alive().should.equal(False) manager.worker_children[1].is_alive().should.equal(False) manager.start() manager.worker_children[0].is_alive().should.equal(True) manager.worker_children[1].is_alive().should.equal(True) manager.stop() manager.worker_children[0].is_alive().should.equal(False) manager.worker_children[1].is_alive().should.equal(False)
def test_manager_start_and_stop(): conn = boto.connect_sqs() conn.create_queue("email") manager = ManagerWorker(queue_prefixes=['email'], worker_concurrency=2) len(manager.worker_children).should.equal(2) manager.worker_children[0].is_alive().should.equal(False) manager.worker_children[1].is_alive().should.equal(False) manager.start() manager.worker_children[0].is_alive().should.equal(True) manager.worker_children[1].is_alive().should.equal(True) manager.stop() manager.worker_children[0].is_alive().should.equal(False) manager.worker_children[1].is_alive().should.equal(False)
def test_master_spawns_worker_processes(): """ Test managing process creates child workers """ # Setup SQS Queue conn = boto3.client('sqs', region_name='us-east-1') conn.create_queue(QueueName="tester") # Setup Manager manager = ManagerWorker(["tester"], 1, 1, 10) manager.start() # Check Workers len(manager.reader_children).should.equal(1) len(manager.worker_children).should.equal(1) manager.reader_children[0].is_alive().should.be.true manager.worker_children[0].is_alive().should.be.true # Cleanup manager.stop()
def test_master_spawns_worker_processes(): """ Test managing process creates child workers """ # Setup SQS Queue conn = boto.connect_sqs() conn.create_queue("tester") # Setup Manager manager = ManagerWorker(["tester"], 1) manager.start() # Check Workers len(manager.reader_children).should.equal(1) len(manager.worker_children).should.equal(1) manager.reader_children[0].is_alive().should.be.true manager.worker_children[0].is_alive().should.be.true # Cleanup manager.stop()
def test_master_shuts_down_busy_process_workers(): """ Test managing process properly cleans up busy Process Workers """ # For debugging test import sys logger = logging.getLogger("pyqs") logger.setLevel(logging.DEBUG) stdout_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stdout_handler) # Setup SQS Queue conn = boto3.client('sqs', region_name='us-east-1') queue_url = conn.create_queue(QueueName="tester")['QueueUrl'] # Add Slow tasks message = json.dumps({ 'task': 'tests.tasks.sleeper', 'args': [], 'kwargs': { 'message': 5, }, }) # Fill the queue (we need a lot of messages to trigger the bug) for _ in range(20): conn.send_message(QueueUrl=queue_url, MessageBody=message) # Create function to watch and kill stuck processes def sleep_and_kill(pid): import os import signal import time # This sleep time is long enoug for 100 messages in queue time.sleep(5) try: os.kill(pid, signal.SIGKILL) except OSError: # Return that we didn't need to kill the process return True else: # Return that we needed to kill the process return False # Setup Manager manager = ManagerWorker(queue_prefixes=["tester"], worker_concurrency=1, interval=0.0, batchsize=1) manager.start() # Give our processes a moment to start time.sleep(1) # Setup Threading watcher try: # Try Python 2 Style thread = ThreadWithReturnValue2( target=sleep_and_kill, args=(manager.reader_children[0].pid, )) thread.daemon = True except TypeError: # Use Python 3 Style thread = ThreadWithReturnValue3( target=sleep_and_kill, args=(manager.reader_children[0].pid, ), daemon=True) thread.start() # Stop the Master Process manager.stop() # Check if we had to kill the Reader Worker or it exited gracefully return_value = thread.join() if not return_value: raise Exception("Reader Worker failed to quit!")
def test_master_shuts_down_busy_process_workers(): """ Test managing process properly cleans up busy Process Workers """ # For debugging test import sys logger = logging.getLogger("pyqs") logger.setLevel(logging.DEBUG) stdout_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stdout_handler) # Setup SQS Queue conn = boto3.client('sqs', region_name='us-east-1') queue_url = conn.create_queue(QueueName="tester")['QueueUrl'] # Add Slow tasks message = json.dumps({ 'task': 'tests.tasks.sleeper', 'args': [], 'kwargs': { 'message': 5, }, }) # Fill the queue (we need a lot of messages to trigger the bug) for _ in range(20): conn.send_message(QueueUrl=queue_url, MessageBody=message) # Create function to watch and kill stuck processes def sleep_and_kill(pid): import os import signal import time # This sleep time is long enoug for 100 messages in queue time.sleep(5) try: os.kill(pid, signal.SIGKILL) except OSError: # Return that we didn't need to kill the process return True else: # Return that we needed to kill the process return False # Setup Manager manager = ManagerWorker( queue_prefixes=["tester"], worker_concurrency=1, interval=0.0, batchsize=1, ) manager.start() # Give our processes a moment to start time.sleep(1) # Setup Threading watcher try: # Try Python 2 Style thread = ThreadWithReturnValue2( target=sleep_and_kill, args=(manager.reader_children[0].pid,)) thread.daemon = True except TypeError: # Use Python 3 Style thread = ThreadWithReturnValue3( target=sleep_and_kill, args=(manager.reader_children[0].pid,), daemon=True, ) thread.start() # Stop the Master Process manager.stop() # Check if we had to kill the Reader Worker or it exited gracefully return_value = thread.join() if not return_value: raise Exception("Reader Worker failed to quit!")