def test_queue_processor_shutsdown_cleanly():
task_queue = queue.Queue()
queue_processor = QueueProcessor()
def process_item(item):
return None
queue_processor.start(queue_=task_queue, process_item=process_item)
assert queue_processor.is_running(), "The QueueProcessor is expected to run while waiting for a task"
queue_processor.schedule_stop()
queue_processor.wait_stop()
assert not queue_processor.is_running(), "The QueueProcessor was stopped - why is it still running?"
def test_queue_processor_processes_jobs():
# Fill task queue
task_queue = queue.Queue()
test_string_1 = "Just testing"
test_string_2 = "Here also"
task_queue.put(test_string_1)
task_queue.put(test_string_2)
# Define handler that puts items to `handled_queue`
handled_queue = queue.Queue()
def process_item(item):
handled_queue.put(item)
# Start processing
queue_processor = QueueProcessor()
try:
queue_processor.start(queue_=task_queue, process_item=process_item)
item = handled_queue.get(block=True)
assert item == test_string_1, "The first item to handle was '{}'!".format(test_string_1)
item2 = handled_queue.get(block=True)
assert item2 == test_string_2, "The second item to handle was '{}'!".format(test_string_2)
assert task_queue.empty(), "There were only two items to handle, why is the queue not empty?"
finally:
queue_processor.schedule_stop()
queue_processor.wait_stop()
def get_scheduler(with_notifier=False):
queue_processor = QueueProcessor()
if with_notifier:
scheduler = Scheduler(queue_processor, notifier=MockNotifier())
else:
scheduler = Scheduler(queue_processor=queue_processor)
return scheduler
def test_find_job_id_precedence(cleanup): # pylint:disable=unused-argument,redefined-outer-name
assert_msg1 = "`find_job_id` should look for job ID by giving precedence to looking by UUID, then by number, then " \
"by pattern; One of the possible 7 combinations failed this precedence test."
assert_msg2 = "`find_job_id` should return `None` as none of the possible arguments matched against submitted job"
service = create_autospec(Service).return_value
scheduler = Scheduler(QueueProcessor())
api = Api(scheduler, service)
job = __get_job(sleep_duration=1)
with scheduler:
scheduler.submit_job(job)
wait_for_true(lambda: job.status == JobStatus.FINISHED)
pat = "{pat}*".format(pat=job.name[:2])
# Test precedence via different valid/invalid combinations
assert api.find_job_id(job_id=job.id, job_number=job.number, pattern=pat) == \
api.find_job_id(job_id=job.id, job_number=job.number, pattern="ohnnoes*") == \
api.find_job_id(job_id=job.id, job_number=job.number + 1, pattern=pat) == \
api.find_job_id(job_id=job.id, job_number=job.number + 1, pattern="boom?") ==\
api.find_job_id(job_id=uuid.uuid4(), job_number=job.number, pattern=pat) == \
api.find_job_id(job_id=uuid.uuid4(), job_number=job.number, pattern="ohnnoes*") == \
api.find_job_id(job_id=uuid.uuid4(), job_number=job.number + 1, pattern=pat) == job.id, assert_msg1
assert api.find_job_id(job_id=uuid.uuid4(),
job_number=job.number + 1,
pattern="boom!") is None, assert_msg2
def test_notification_history_no_notifier(cleanup): # pylint:disable=unused-argument,redefined-outer-name
scheduler = Scheduler(QueueProcessor())
service = create_autospec(Service).return_value
api = Api(scheduler, service)
job = __get_job(sleep_duration=1)
notification_history = api.get_notification_history(job.id)
assert dict(
) == notification_history, "Without notifiers, notification history is expected to be empty"
def test_get_notification_status_empty(cleanup): # pylint:disable=unused-argument,redefined-outer-name
scheduler = Scheduler(QueueProcessor())
service = create_autospec(Service).return_value
api = Api(scheduler, service)
job = __get_job(sleep_duration=1)
assert api.get_notification_status(
job.id
) == "", "Without notifiers, notification status is expected to be \"\""
def test_find_job_no_input(cleanup): # pylint:disable=unused-argument,redefined-outer-name
service = create_autospec(Service).return_value
scheduler = Scheduler(QueueProcessor())
api = Api(scheduler, service)
job = __get_job(sleep_duration=1)
with scheduler:
scheduler.submit_job(job)
wait_for_true(lambda: job.status == JobStatus.FINISHED)
assert api.find_job_id(
) is None, "Call without arguments should fail silently by returning `None`"
def mock_api(mock_sagemaker_job_monitor):
notifier = create_autospec(Notifier).return_value
mock_event_loop = create_autospec(asyncio.AbstractEventLoop).return_value
scheduler = Scheduler(QueueProcessor(),
notifier=notifier,
event_loop=mock_event_loop)
service = create_autospec(Service).return_value
yield Api(scheduler=scheduler,
service=service,
notifier=notifier,
sagemaker_job_monitor=mock_sagemaker_job_monitor)
def test_find_job_id_by_number(cleanup): # pylint:disable=unused-argument,redefined-outer-name
service = create_autospec(Service).return_value
scheduler = Scheduler(QueueProcessor())
api = Api(scheduler, service)
job = __get_job(sleep_duration=1)
with scheduler:
scheduler.submit_job(job)
wait_for_true(lambda: job.status == JobStatus.FINISHED)
assert api.find_job_id(
job_number=job.number) == job.id, "Job ID should match real job ID"
assert api.find_job_id(job_number=job.number + 1) is None, "Invalid job number should return None as no matching " \
"job exists"
def test_get_job_output(cleanup): # pylint:disable=unused-argument,redefined-outer-name
service = create_autospec(Service).return_value
scheduler = Scheduler(QueueProcessor())
api = Api(scheduler, service)
job = __get_job(sleep_duration=1)
with scheduler:
scheduler.submit_job(job)
wait_for_true(lambda: job.status == JobStatus.FINISHED)
output_path, stderr, stdout = api.get_job_output(job.id)
assert output_path == job.output_path, "Expected to return internal job output path"
assert stderr == job.stderr, "Expected to return internal job stderr path"
assert stdout == job.stdout, "Expected to return internal job stdout path"
def test_find_job_id_by_pattern(cleanup): # pylint:disable=unused-argument,redefined-outer-name
service = create_autospec(Service).return_value
scheduler = Scheduler(QueueProcessor())
api = Api(scheduler, service)
job = __get_job(sleep_duration=1)
with scheduler:
scheduler.submit_job(job)
wait_for_true(lambda: job.status == JobStatus.FINISHED)
pat = "{pat}*".format(pat=job.name[:2])
assert api.find_job_id(
pattern=pat) == job.id, "Job ID should match real job ID"
assert api.find_job_id(pattern="ohnoes*") is None, "Pattern should not match submitted job name and should return " \
"`None`"
def test_get_job(cleanup): # pylint:disable=unused-argument,redefined-outer-name
service = create_autospec(Service).return_value
scheduler = Scheduler(QueueProcessor())
api = Api(scheduler, service)
job = __get_job(sleep_duration=1)
assert api.get_job(
job.id
) is None, "Job has not been submitted to scheduler yet, return value should be `None`"
scheduler.submit_job(job)
assert api.get_job(
job.id
) == job, "Job has been submitted to scheduler, returned value should match submitted job"
def _build_api(service, cloud_client): # pylint: disable=too-many-locals
# Cyclic import check is cancelled here; pylint complains about it even though it's only imported in the daemon
# process, which is a clean one and therefore there are no cyclic imports.
from meeshkan.core.api import Api # pylint: disable=cyclic-import
from meeshkan.notifications.notifiers import CloudNotifier, LoggingNotifier, NotifierCollection
from meeshkan.core.tasks import TaskPoller
from meeshkan.core.scheduler import Scheduler, QueueProcessor
from meeshkan.core.config import ensure_base_dirs as ensure_base_dirs_
from meeshkan.core.logger import setup_logging as setup_logging_
from meeshkan.core.sagemaker_monitor import SageMakerJobMonitor
ensure_base_dirs_()
setup_logging_(silent=True)
cloud_notifier = CloudNotifier(
name="Cloud Service",
post_payload=cloud_client.post_payload,
upload_file=cloud_client.post_payload_with_file)
logging_notifier = LoggingNotifier(name="Local Service")
task_poller = TaskPoller(cloud_client.pop_tasks)
queue_processor = QueueProcessor()
notifier_collection = NotifierCollection(
*[cloud_notifier, logging_notifier])
scheduler = Scheduler(queue_processor=queue_processor,
notifier=notifier_collection)
sagemaker_job_monitor = SageMakerJobMonitor(
notify_start=notifier_collection.notify_job_start,
notify_update=notifier_collection.notify,
notify_finish=notifier_collection.notify_job_end)
api = Api(scheduler=scheduler,
service=service,
task_poller=task_poller,
notifier=notifier_collection,
sagemaker_job_monitor=sagemaker_job_monitor)
api.add_stop_callback(cloud_client.close)
return api
def test_notification_history_with_notifier(cleanup): # pylint:disable=unused-argument,redefined-outer-name
notifier = MockNotifier()
service = create_autospec(Service).return_value
scheduler = Scheduler(QueueProcessor(), notifier=notifier)
api = Api(scheduler, service, notifier=notifier)
job = __get_job(sleep_duration=1)
with scheduler: # calls .start() and .stop()
scheduler.submit_job(job)
wait_for_true(lambda: job.status == JobStatus.FINISHED)
notification_history = api.get_notification_history(job.id)
assert len(
notification_history
) == 1, "There is only one notifier (e.g. single key in `history`)"
assert len(
notification_history[notifier.name]
) == 2, "There should be two events registered! (job start, job end)"
assert type(
notification_history[notifier.name]
[0]) == str, "Notification history items are expected to be strings"
async def test_stopping_job_with_task(cleanup): # pylint:disable=unused-argument,redefined-outer-name
scheduler = Scheduler(QueueProcessor())
service = create_autospec(Service).return_value
api = Api(scheduler, service)
job = __get_job()
with scheduler: # calls .start() and .stop()
scheduler.submit_job(job)
wait_for_true(lambda: job.status == JobStatus.RUNNING)
# Schedule stop job task
loop = asyncio.get_event_loop()
loop.create_task(
api.handle_task(
TaskFactory.build({
'__typename': 'StopJobTask',
'job': {
'job_id': job.id
}
})))
wait_for_true(scheduler._job_queue.empty)
assert job.status in [JobStatus.CANCELLED_BY_USER, JobStatus.CANCELED]