def test_cloud_notifier_job_update_existing_file(self, job): # pylint:disable=unused-argument,redefined-outer-name
"""Tests the job update for CloudNotifier when the image exists and is valid"""
posted_payload = {}
def fake_post(payload):
nonlocal posted_payload
posted_payload = payload
def fake_upload(image_path, download_link):
assert image_path == __file__, "Expecting given mock `image_path` to match '{}'".format(
__file__)
return "no_upload"
cloud_notifier = CloudNotifier(fake_post, fake_upload)
cloud_notifier.notify(job, __file__, n_iterations=-1)
expected_payload = {
'query':
'mutation NotifyJobEvent($in: JobScalarChangesWithImageInput!)'
' {notifyJobScalarChangesWithImage(input: $in)}'
}
# Verify query structure
assert 'query' in posted_payload, "Payload is expected to contain the key 'query'"
assert posted_payload['query'] == expected_payload[
'query'], "GraphQL queries should be identical"
assert 'variables' in posted_payload, "Payload is expected to contain the key 'variables'"
assert 'in' in posted_payload[
'variables'], "'variables' dictionary is expected to contain a key called 'in'"
# Verify empty imageUrl
variables = posted_payload['variables']['in']
assert 'imageUrl' in variables, "'in' dictionary is expected to contain a key called 'imageUrl'"
assert variables['imageUrl'] == "no_upload", "`imageUrl` value is expected to " \
"contain hard-coded value `no_upload`"
def test_notifier_collection_registering_notifiers(self): # pylint:disable=unused-argument,redefined-outer-name
"""Tests the register_notifier method"""
cloud_notifier = CloudNotifier(_empty_post, _empty_upload)
cloud_notifier2 = CloudNotifier(_empty_post,
_empty_upload,
name="smeagol")
collection = NotifierCollection()
# Sanity check
assert len(collection._notifiers
) == 0, "Sanity check - empty NotifierCollection"
# Adding
assert collection.register_notifier(
cloud_notifier), "Adding a new notifier should return True"
assert len(collection._notifiers
) == 1, "A single notifier was added to the collection!"
# Adding again?
assert not collection.register_notifier(cloud_notifier), "Trying to add an already-added notifier should " \
"return False and not add the notifier to the list"
# Adding same class, different name
assert collection.register_notifier(
cloud_notifier2
), "Adding a new notifier of the same class should succeed"
assert len(collection._notifiers
) == 2, "There are now two registered notifiers!"
def test_cloud_notifier_job_update_no_image(self): # pylint:disable=unused-argument,redefined-outer-name
"""Tests the job update for CloudNotifier when no image is available or image does not exist"""
posted_payload = {}
def fake_post(payload):
nonlocal posted_payload
posted_payload = payload
cloud_notifier = CloudNotifier(fake_post, _empty_upload)
for path in ["fake_path", None]:
cloud_notifier.notify(_get_job(), path, n_iterations=-1)
expected_payload = {
'query':
'mutation NotifyJobEvent($in: JobScalarChangesWithImageInput!)'
' {notifyJobScalarChangesWithImage(input: $in)}'
}
# Verify query structure
assert 'query' in posted_payload, "Payload is expected to contain the key 'query'"
assert posted_payload['query'] == expected_payload[
'query'], "GraphQL queries should be identical"
assert 'variables' in posted_payload, "Payload is expected to contain the key 'variables'"
assert 'in' in posted_payload['variables'], "'variables' dictionary is expected " \
"to contain a key called 'in'"
# Verify empty imageUrl
variables = posted_payload['variables']['in']
assert 'imageUrl' in variables, "'in' dictionary is expected to contain a key called 'imageUrl'"
assert variables[
'imageUrl'] == '', "`imageUrl` value is expected to be empty"
def test_notifier_history(self): # pylint:disable=unused-argument,redefined-outer-name
"""Tests the Notifier's built-in history management via CloudNotifier"""
should_raise = False
def _fake_post(payload):
nonlocal should_raise
if should_raise:
raise RuntimeError
cloud_notifier = CloudNotifier(_fake_post, _empty_upload)
job = _get_job()
cloud_notifier.notify_job_start(job)
# Validate history for notify_job_start
notifications = cloud_notifier.get_notification_history(
job.id)[cloud_notifier.name]
assert len(
notifications
) == 1, "Expecting a single notification at this point (JobStart)"
assert notifications[
0].type == NotificationType.JOB_START, "Notification type should be Job Start"
assert notifications[0].status == NotificationStatus.SUCCESS, "Notification is expected to succeed " \
"with Mock `post`"
last_notification = cloud_notifier.get_last_notification_status(
job.id)[cloud_notifier.name]
assert last_notification is not None, "There was a single notification, we expect a concrete value"
assert last_notification.type == NotificationType.JOB_START, "Notification type should be Job Start"
assert last_notification.status == NotificationStatus.SUCCESS, "Notification was expected to succeed"
should_raise = True
cloud_notifier.notify_job_end(job)
# Validate history for notify_job_end
notifications = cloud_notifier.get_notification_history(
job.id)[cloud_notifier.name]
assert len(
notifications
) == 2, "Expecting two notification at this point (JobStart, JobEnd)"
assert notifications[
1].type == NotificationType.JOB_END, "Second notification type should be Job End"
assert notifications[
1].status == NotificationStatus.FAILED, "Notification has failed here due to set flag"
last_notification = cloud_notifier.get_last_notification_status(
job.id)[cloud_notifier.name]
assert last_notification is not None, "We expect a concrete value after any event has happened"
assert last_notification.type == NotificationType.JOB_END, "Last notification type was Job End"
assert last_notification.status == NotificationStatus.FAILED, "Last notification has failed to due to set flag"
def test_cloud_notifier_notifies_failed_job_with_correct_payload(self):
fake_post = MagicMock()
fake_upload = MagicMock()
cloud_notifier = CloudNotifier(fake_post, fake_upload)
def get_failed_job():
job_id = uuid.uuid4()
resource_dir = Path(os.path.dirname(__file__)).joinpath(
'resources', 'logs')
job_ = Job(Executable(output_path=resource_dir),
job_number=0,
job_uuid=job_id)
job_.status = JobStatus.FAILED
return job_
failed_job = get_failed_job()
cloud_notifier.notify_job_end(failed_job)
cloud_notification_status = cloud_notifier.get_last_notification_status(
failed_job.id)["CloudNotifier"]
assert cloud_notification_status.status == NotificationStatus.SUCCESS, "Notification status should be success"
# Posted payload
fake_post.assert_called_once()
args, _ = fake_post.call_args
assert len(
args
) == 1, "Expected mock post to have been called with one argument"
payload = args[0]
assert payload is not None, "Expected posted payload to not be None"
# Query
assert "query" in payload, "Expected posted payload to have query field"
query = payload["query"]
assert query is not None
# Variables
assert "variables" in payload, "Expected posted payload to have variables field"
variables = payload["variables"]
assert variables is not None
assert "in" in variables, "Expected variables to have 'in' field"
inp = variables["in"]
stderr = inp["stderr"]
assert stderr is not None
assert len(stderr) > 50
assert inp["job_id"] is not None
def test_cloud_notifier_job_start_end_queries(self, job): # pylint:disable=redefined-outer-name
"""Tests the job start/end for CloudNotifier"""
posted_payload = {}
def fake_post(payload):
nonlocal posted_payload
posted_payload = payload
# Initializations (sanity checks)
assert posted_payload == {}, "Sanity test - `posted_payload` dictionary should be empty at this point"
cloud_notifier = CloudNotifier(fake_post, _empty_upload)
expected_payload_start = {
"query":
"mutation NotifyJobStart($in: JobStartInput!) { notifyJobStart(input: $in) }"
}
expected_payload_end = {
"query":
"mutation NotifyJobEnd($in: JobDoneInput!) { notifyJobDone(input: $in) }"
}
notifications = cloud_notifier.get_notification_history(
job.id)[cloud_notifier.name]
assert len(
notifications
) == 0, "Expecting no notification at this point as no events have occurred"
last_notification = cloud_notifier.get_last_notification_status(
job.id)[cloud_notifier.name]
assert last_notification is None, "There is no last notification either, expected `None` value"
cloud_notifier.notify_job_start(job)
# Validate query for notify_job_start
assert "query" in posted_payload, "Payload is expected to contain the key 'query'"
assert posted_payload["query"] == expected_payload_start[
"query"], "GraphQL queries should be identical"
cloud_notifier.notify_job_end(job)
# Validate query for notify_job_end
assert "query" in posted_payload, "Payload is expected to contain the key 'query'"
assert posted_payload["query"] == expected_payload_end[
"query"], "GraphQL queries should be identical"
assert "variables" in posted_payload, "Posted payload is expected to contain a 'variables' key"
variables = posted_payload["variables"]
assert "in" in variables, "'variables' dictionary is expected to contain a key called 'in'"
def _build_api(service, cloud_client): # pylint: disable=too-many-locals
# Cyclic import check is cancelled here; pylint complains about it even though it's only imported in the daemon
# process, which is a clean one and therefore there are no cyclic imports.
from meeshkan.core.api import Api # pylint: disable=cyclic-import
from meeshkan.notifications.notifiers import CloudNotifier, LoggingNotifier, NotifierCollection
from meeshkan.core.tasks import TaskPoller
from meeshkan.core.scheduler import Scheduler, QueueProcessor
from meeshkan.core.config import ensure_base_dirs as ensure_base_dirs_
from meeshkan.core.logger import setup_logging as setup_logging_
from meeshkan.core.sagemaker_monitor import SageMakerJobMonitor
ensure_base_dirs_()
setup_logging_(silent=True)
cloud_notifier = CloudNotifier(
name="Cloud Service",
post_payload=cloud_client.post_payload,
upload_file=cloud_client.post_payload_with_file)
logging_notifier = LoggingNotifier(name="Local Service")
task_poller = TaskPoller(cloud_client.pop_tasks)
queue_processor = QueueProcessor()
notifier_collection = NotifierCollection(
*[cloud_notifier, logging_notifier])
scheduler = Scheduler(queue_processor=queue_processor,
notifier=notifier_collection)
sagemaker_job_monitor = SageMakerJobMonitor(
notify_start=notifier_collection.notify_job_start,
notify_update=notifier_collection.notify,
notify_finish=notifier_collection.notify_job_end)
api = Api(scheduler=scheduler,
service=service,
task_poller=task_poller,
notifier=notifier_collection,
sagemaker_job_monitor=sagemaker_job_monitor)
api.add_stop_callback(cloud_client.close)
return api
def test_notifier_collection_notifiers_init(self): # pylint:disable=unused-argument,redefined-outer-name
"""Tests init with notifiers"""
assert_msg1 = "Both created notifiers should be found in the `_notifiers` list"
cloud_notifier = CloudNotifier(_empty_post, _empty_upload)
logging_notifier = LoggingNotifier()
# Proper init
collection = NotifierCollection(*[cloud_notifier, logging_notifier])
assert len(
collection._notifiers
) == 2, "There are two registered notifiers in the collection"
assert cloud_notifier in collection._notifiers and logging_notifier in collection._notifiers, assert_msg1
# Empty init
collection = NotifierCollection()
assert len(collection._notifiers) == 0, "NotifierCollection was instantiated without any notifiers! " \
"How come there are any registered?"
# Bad init
collection = NotifierCollection(*[cloud_notifier, cloud_notifier])
assert len(collection._notifiers) == 1, "NotifierCollection was instantiated with the same object multiple" \
"times, but only one unique instance of an object can be stored."
def test_notifier_collection_notifications(self): # pylint:disable=unused-argument,redefined-outer-name
"""Tests the job notifications are sent to all registered notifiers, along with history management"""
assert_msg1 = "Last notification type was JobUpdate"
assert_msg2 = "CloudNotifier is expected to succeed with a fake `post` method"
assert_msg3 = "LoggingNotifier is expected to fail with non-existing output path"
assert_msg4 = "History for CloudNotifier from NotifierCollection should match CloudNotifier internal history"
assert_msg5 = "History for LoggingNotifier from NotifierCollection " \
"should match LoggingNotifier internal history"
cloud_counter = 0
logging_counter = 0
def fake_post(payload):
nonlocal cloud_counter
cloud_counter += 1
def fake_log(job_id, message):
nonlocal logging_counter
logging_counter += 1
job = _get_job()
logging_notifier = LoggingNotifier()
with mock.patch.object(logging_notifier, 'log', fake_log):
cloud_notifier = CloudNotifier(fake_post, _empty_upload)
collection = NotifierCollection(
*[cloud_notifier, logging_notifier])
# Test with notify_job_start
collection.notify_job_start(job)
assert cloud_counter == logging_counter == 1, "A single event (JobStart) should be sent to both notifiers"
# Test with notify_job_end
collection.notify_job_end(job)
assert cloud_counter == logging_counter == 2, "Two events have now been registered to both notifiers " \
"(JobStart, JobEnd)"
# Test with notify
collection.notify(job, "", -1)
# Validate `notify` via job_history
last_notification = collection.get_last_notification_status(job.id)
assert len(last_notification) == 2, "There are two notifiers, so we expect two keys " \
"in the last notification"
assert cloud_notifier.name in last_notification, "CloudNotifier name '{}' should be a " \
"key".format(cloud_notifier.name)
assert logging_notifier.name in last_notification, "LoggingNotifier name '{}' should be a " \
"key".format(logging_notifier.name)
assert last_notification[
cloud_notifier.
name].type == NotificationType.JOB_UPDATE, assert_msg1
assert last_notification[
logging_notifier.
name].type == NotificationType.JOB_UPDATE, assert_msg1
# Cloud notification is expected to be successful as we emulate the upload and posting process
assert last_notification[
cloud_notifier.
name].status == NotificationStatus.SUCCESS, assert_msg2
# Logging notification is expected to fail as the target directory does not exist
assert last_notification[
logging_notifier.
name].status == NotificationStatus.FAILED, assert_msg3
# Test history
history = collection.get_notification_history(job.id)
assert len(
history
) == 2, "The entire history should have two keys - one for each notifier"
assert cloud_notifier.name in last_notification, "CloudNotifier name '{}' should be a " \
"key".format(cloud_notifier.name)
assert logging_notifier.name in last_notification, "LoggingNotifier name '{}' should be a " \
"key".format(logging_notifier.name)
cloud_history = history[cloud_notifier.name]
logging_history = history[logging_notifier.name]
assert cloud_history == cloud_notifier.get_notification_history(
job.id)[cloud_notifier.name], assert_msg4
assert logging_history == logging_notifier.get_notification_history(
job.id)[logging_notifier.name], assert_msg5