def test_find_job_id_precedence(cleanup): # pylint:disable=unused-argument,redefined-outer-name assert_msg1 = "`find_job_id` should look for job ID by giving precedence to looking by UUID, then by number, then " \ "by pattern; One of the possible 7 combinations failed this precedence test." assert_msg2 = "`find_job_id` should return `None` as none of the possible arguments matched against submitted job" service = create_autospec(Service).return_value scheduler = Scheduler(QueueProcessor()) api = Api(scheduler, service) job = __get_job(sleep_duration=1) with scheduler: scheduler.submit_job(job) wait_for_true(lambda: job.status == JobStatus.FINISHED) pat = "{pat}*".format(pat=job.name[:2]) # Test precedence via different valid/invalid combinations assert api.find_job_id(job_id=job.id, job_number=job.number, pattern=pat) == \ api.find_job_id(job_id=job.id, job_number=job.number, pattern="ohnnoes*") == \ api.find_job_id(job_id=job.id, job_number=job.number + 1, pattern=pat) == \ api.find_job_id(job_id=job.id, job_number=job.number + 1, pattern="boom?") ==\ api.find_job_id(job_id=uuid.uuid4(), job_number=job.number, pattern=pat) == \ api.find_job_id(job_id=uuid.uuid4(), job_number=job.number, pattern="ohnnoes*") == \ api.find_job_id(job_id=uuid.uuid4(), job_number=job.number + 1, pattern=pat) == job.id, assert_msg1 assert api.find_job_id(job_id=uuid.uuid4(), job_number=job.number + 1, pattern="boom!") is None, assert_msg2
def test_api_submits_job(cleanup): # pylint:disable=unused-argument,redefined-outer-name scheduler = create_autospec(Scheduler).return_value service = create_autospec(Service).return_value api = Api(scheduler, service) job_args = ('echo', 'Hello') job = api.submit(job_args) scheduler.submit_job.assert_called_with(job)
def test_does_not_start_monitoring_finished_sagemaker_job(mock_api: Api): # pylint:disable=redefined-outer-name job_name = "pytorch-rnn-2019-01-04-11-20-03" mock_api.sagemaker_job_monitor.create_job.return_value = SageMakerJob( job_name=job_name, status=JobStatus.FINISHED, poll_interval=60) mock_api.monitor_sagemaker(job_name=job_name) mock_api.sagemaker_job_monitor.create_job.assert_called_with( job_name=job_name, poll_interval=None) mock_api.sagemaker_job_monitor.start.assert_not_called()
def test_notification_history_no_notifier(cleanup): # pylint:disable=unused-argument,redefined-outer-name scheduler = Scheduler(QueueProcessor()) service = create_autospec(Service).return_value api = Api(scheduler, service) job = __get_job(sleep_duration=1) notification_history = api.get_notification_history(job.id) assert dict( ) == notification_history, "Without notifiers, notification history is expected to be empty"
def test_get_notification_status_empty(cleanup): # pylint:disable=unused-argument,redefined-outer-name scheduler = Scheduler(QueueProcessor()) service = create_autospec(Service).return_value api = Api(scheduler, service) job = __get_job(sleep_duration=1) assert api.get_notification_status( job.id ) == "", "Without notifiers, notification status is expected to be \"\""
def test_find_job_no_input(cleanup): # pylint:disable=unused-argument,redefined-outer-name service = create_autospec(Service).return_value scheduler = Scheduler(QueueProcessor()) api = Api(scheduler, service) job = __get_job(sleep_duration=1) with scheduler: scheduler.submit_job(job) wait_for_true(lambda: job.status == JobStatus.FINISHED) assert api.find_job_id( ) is None, "Call without arguments should fail silently by returning `None`"
def test_find_job_id_by_number(cleanup): # pylint:disable=unused-argument,redefined-outer-name service = create_autospec(Service).return_value scheduler = Scheduler(QueueProcessor()) api = Api(scheduler, service) job = __get_job(sleep_duration=1) with scheduler: scheduler.submit_job(job) wait_for_true(lambda: job.status == JobStatus.FINISHED) assert api.find_job_id( job_number=job.number) == job.id, "Job ID should match real job ID" assert api.find_job_id(job_number=job.number + 1) is None, "Invalid job number should return None as no matching " \ "job exists"
def test_get_job_output(cleanup): # pylint:disable=unused-argument,redefined-outer-name service = create_autospec(Service).return_value scheduler = Scheduler(QueueProcessor()) api = Api(scheduler, service) job = __get_job(sleep_duration=1) with scheduler: scheduler.submit_job(job) wait_for_true(lambda: job.status == JobStatus.FINISHED) output_path, stderr, stdout = api.get_job_output(job.id) assert output_path == job.output_path, "Expected to return internal job output path" assert stderr == job.stderr, "Expected to return internal job stderr path" assert stdout == job.stdout, "Expected to return internal job stdout path"
def test_find_job_id_by_pattern(cleanup): # pylint:disable=unused-argument,redefined-outer-name service = create_autospec(Service).return_value scheduler = Scheduler(QueueProcessor()) api = Api(scheduler, service) job = __get_job(sleep_duration=1) with scheduler: scheduler.submit_job(job) wait_for_true(lambda: job.status == JobStatus.FINISHED) pat = "{pat}*".format(pat=job.name[:2]) assert api.find_job_id( pattern=pat) == job.id, "Job ID should match real job ID" assert api.find_job_id(pattern="ohnoes*") is None, "Pattern should not match submitted job name and should return " \ "`None`"
def test_get_job(cleanup): # pylint:disable=unused-argument,redefined-outer-name service = create_autospec(Service).return_value scheduler = Scheduler(QueueProcessor()) api = Api(scheduler, service) job = __get_job(sleep_duration=1) assert api.get_job( job.id ) is None, "Job has not been submitted to scheduler yet, return value should be `None`" scheduler.submit_job(job) assert api.get_job( job.id ) == job, "Job has been submitted to scheduler, returned value should match submitted job"
def test_api_as_contextmanager(cleanup): # pylint:disable=unused-argument,redefined-outer-name scheduler = create_autospec(Scheduler).return_value service = create_autospec(Service).return_value api = Api(scheduler, service) with api: scheduler.start.assert_called() service.stop.assert_called() scheduler.stop.assert_called()
def _build_api(service, cloud_client): # pylint: disable=too-many-locals # Cyclic import check is cancelled here; pylint complains about it even though it's only imported in the daemon # process, which is a clean one and therefore there are no cyclic imports. from meeshkan.core.api import Api # pylint: disable=cyclic-import from meeshkan.notifications.notifiers import CloudNotifier, LoggingNotifier, NotifierCollection from meeshkan.core.tasks import TaskPoller from meeshkan.core.scheduler import Scheduler, QueueProcessor from meeshkan.core.config import ensure_base_dirs as ensure_base_dirs_ from meeshkan.core.logger import setup_logging as setup_logging_ from meeshkan.core.sagemaker_monitor import SageMakerJobMonitor ensure_base_dirs_() setup_logging_(silent=True) cloud_notifier = CloudNotifier( name="Cloud Service", post_payload=cloud_client.post_payload, upload_file=cloud_client.post_payload_with_file) logging_notifier = LoggingNotifier(name="Local Service") task_poller = TaskPoller(cloud_client.pop_tasks) queue_processor = QueueProcessor() notifier_collection = NotifierCollection( *[cloud_notifier, logging_notifier]) scheduler = Scheduler(queue_processor=queue_processor, notifier=notifier_collection) sagemaker_job_monitor = SageMakerJobMonitor( notify_start=notifier_collection.notify_job_start, notify_update=notifier_collection.notify, notify_finish=notifier_collection.notify_job_end) api = Api(scheduler=scheduler, service=service, task_poller=task_poller, notifier=notifier_collection, sagemaker_job_monitor=sagemaker_job_monitor) api.add_stop_callback(cloud_client.close) return api
def mock_api(mock_sagemaker_job_monitor): notifier = create_autospec(Notifier).return_value mock_event_loop = create_autospec(asyncio.AbstractEventLoop).return_value scheduler = Scheduler(QueueProcessor(), notifier=notifier, event_loop=mock_event_loop) service = create_autospec(Service).return_value yield Api(scheduler=scheduler, service=service, notifier=notifier, sagemaker_job_monitor=mock_sagemaker_job_monitor)
def test_notification_history_with_notifier(cleanup): # pylint:disable=unused-argument,redefined-outer-name notifier = MockNotifier() service = create_autospec(Service).return_value scheduler = Scheduler(QueueProcessor(), notifier=notifier) api = Api(scheduler, service, notifier=notifier) job = __get_job(sleep_duration=1) with scheduler: # calls .start() and .stop() scheduler.submit_job(job) wait_for_true(lambda: job.status == JobStatus.FINISHED) notification_history = api.get_notification_history(job.id) assert len( notification_history ) == 1, "There is only one notifier (e.g. single key in `history`)" assert len( notification_history[notifier.name] ) == 2, "There should be two events registered! (job start, job end)" assert type( notification_history[notifier.name] [0]) == str, "Notification history items are expected to be strings"
async def test_stopping_job_with_task(cleanup): # pylint:disable=unused-argument,redefined-outer-name scheduler = Scheduler(QueueProcessor()) service = create_autospec(Service).return_value api = Api(scheduler, service) job = __get_job() with scheduler: # calls .start() and .stop() scheduler.submit_job(job) wait_for_true(lambda: job.status == JobStatus.RUNNING) # Schedule stop job task loop = asyncio.get_event_loop() loop.create_task( api.handle_task( TaskFactory.build({ '__typename': 'StopJobTask', 'job': { 'job_id': job.id } }))) wait_for_true(scheduler._job_queue.empty) assert job.status in [JobStatus.CANCELLED_BY_USER, JobStatus.CANCELED]
def test_api_stop_callbacks(cleanup): # pylint:disable=unused-argument,redefined-outer-name scheduler = create_autospec(Scheduler).return_value service = create_autospec(Service).return_value api = Api(scheduler, service) callback_called = False def callback(): nonlocal callback_called callback_called = True api.add_stop_callback(callback) api.stop() assert callback_called, "Callback is expected to be called after calling `stop`" service.stop.assert_called() scheduler.stop.assert_called()