def test_ask_with_real_run_function_with_no_log_message_forwarding(self):
"""Test that a service can ask a question to another service that is serving and receive an answer. Use a real
run function rather than a mock so that the underlying `Runner` instance is used, and check that remote log
messages aren't forwarded to the local logger.
"""
child = MockService(backend=BACKEND,
run_function=create_run_function())
parent = MockService(backend=BACKEND, children={child.id: child})
with self.assertLogs() as logging_context:
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
answer = self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={},
subscribe_to_logs=False,
)
self.assertEqual(
answer,
{
"output_values": MockAnalysis().output_values,
"output_manifest": MockAnalysis().output_manifest
},
)
self.assertTrue(
all("[REMOTE]" not in message
for message in logging_context.output))
def test_child_can_ask_its_own_child_questions(self):
"""Test that a child can contact its own child while answering a question from a parent."""
child_of_child = self.make_new_child(
BACKEND,
run_function_returnee=DifferentMockAnalysis(),
use_mock=True)
def child_run_function(analysis_id, input_values, input_manifest,
analysis_log_handler, handle_monitor_message):
subscription, _ = child.ask(service_id=child_of_child.id,
input_values=input_values)
return MockAnalysis(output_values={
input_values["question"]:
child.wait_for_answer(subscription)
})
child = MockService(
backend=BACKEND,
run_function=child_run_function,
children={child_of_child.id: child_of_child},
)
parent = MockService(backend=BACKEND, children={child.id: child})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
child_of_child.serve()
answer = self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={
"question": "What does the child of the child say?"
},
)
self.assertEqual(
answer,
{
"output_values": {
"What does the child of the child say?": {
"output_values": DifferentMockAnalysis.output_values,
"output_manifest":
DifferentMockAnalysis.output_manifest,
}
},
"output_manifest": None,
},
)
def test_ask_with_real_run_function_with_log_message_forwarding(self):
"""Test that a service can ask a question to another service that is serving and receive an answer. Use a real
run function rather than a mock so that the underlying `Runner` instance is used, and check that remote log
messages are forwarded to the local logger.
"""
child = MockService(backend=BACKEND,
run_function=create_run_function())
parent = MockService(backend=BACKEND, children={child.id: child})
with self.assertLogs() as logs_context_manager:
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
answer = self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={},
subscribe_to_logs=True,
service_name="my-super-service",
)
self.assertEqual(
answer,
{
"output_values": MockAnalysis().output_values,
"output_manifest": MockAnalysis().output_manifest
},
)
# Check that the two expected remote log messages were logged consecutively in the right order with the service
# name added as context at the start of the messages.
start_remote_analysis_message_present = False
finish_remote_analysis_message_present = False
for i, log_record in enumerate(logs_context_manager.records):
if "[my-super-service" in log_record.msg and "Starting analysis." in log_record.msg:
start_remote_analysis_message_present = True
if ("[my-super-service"
in logs_context_manager.records[i + 1].msg
and "Finished analysis."
in logs_context_manager.records[i + 1].msg):
finish_remote_analysis_message_present = True
break
self.assertTrue(start_remote_analysis_message_present)
self.assertTrue(finish_remote_analysis_message_present)
def test_monitoring_update_fails_if_schema_not_met(self):
"""Test that an error is raised and sent to the analysis logger if a monitor message fails schema validation,
but earlier valid monitor messages still make it to the parent's monitoring callback.
"""
def create_run_function_with_monitoring():
def mock_app(analysis):
analysis.send_monitor_message(
{"status": "my first monitor message"})
analysis.send_monitor_message(
{"wrong": "my second monitor message"})
analysis.send_monitor_message(
{"status": "my third monitor message"})
twine = """
{
"input_values_schema": {"type": "object", "required": []},
"monitor_message_schema": {
"type": "object",
"properties": {"status": {"type": "string"}},
"required": ["status"]
}
}
"""
return Runner(app_src=mock_app, twine=twine).run
child = MockService(backend=BACKEND,
run_function=create_run_function_with_monitoring())
parent = MockService(backend=BACKEND, children={child.id: child})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
subscription, _ = parent.ask(child.id, input_values={})
monitoring_data = []
with self.assertRaises(InvalidMonitorMessage):
parent.wait_for_answer(
subscription,
handle_monitor_message=lambda data: monitoring_data
.append(data),
)
self.assertEqual(
monitoring_data,
[{
"status": "my first monitor message"
}],
)
def test_ask_with_forwarding_exception_log_message(self):
"""Test that exception/error logs are forwarded to the asker successfully."""
def create_exception_logging_run_function():
def mock_app(analysis):
try:
raise OSError("This is an OSError.")
except OSError:
logger.exception(
"An example exception to log and forward to the parent."
)
return Runner(
app_src=mock_app,
twine=
'{"input_values_schema": {"type": "object", "required": []}}'
).run
child = MockService(
backend=BACKEND,
run_function=create_exception_logging_run_function())
parent = MockService(backend=BACKEND, children={child.id: child})
with self.assertLogs() as logs_context_manager:
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={},
subscribe_to_logs=True,
service_name="my-super-service",
)
error_logged = False
for record in logs_context_manager.records:
if (record.levelno == logging.ERROR and
"An example exception to log and forward to the parent."
in record.message
and "This is an OSError" in record.exc_text):
error_logged = True
break
self.assertTrue(error_logged)
def test_ask_with_output_manifest(self):
"""Test that a service can receive an output manifest as part of the answer to a question."""
child = self.make_new_child(
BACKEND,
run_function_returnee=MockAnalysisWithOutputManifest(),
use_mock=True)
parent = MockService(backend=BACKEND, children={child.id: child})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
answer = self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={},
)
self.assertEqual(answer["output_values"],
MockAnalysisWithOutputManifest.output_values)
self.assertEqual(answer["output_manifest"].id,
MockAnalysisWithOutputManifest.output_manifest.id)
def test_unknown_exceptions_in_responder_are_handled_and_sent_to_asker(
self):
"""Test that exceptions not in the exceptions mapping are simply raised as `Exception`s by the asker."""
class AnUnknownException(Exception):
pass
child = self.make_child_service_with_error(
AnUnknownException("This is an exception unknown to the asker."))
parent = MockService(backend=BACKEND, children={child.id: child})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
with self.assertRaises(Exception) as context:
self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={},
)
self.assertEqual(
type(context.exception).__name__, "AnUnknownException")
self.assertIn("This is an exception unknown to the asker.",
context.exception.args[0])
def test_exceptions_with_multiple_arguments_in_responder_are_handled_and_sent_to_asker(
self):
"""Test that exceptions with multiple arguments raised in the child service are handled and sent back to
the asker.
"""
child = self.make_child_service_with_error(
FileNotFoundError(2, "No such file or directory: 'blah'"))
parent = MockService(backend=BACKEND, children={child.id: child})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
with self.assertRaises(FileNotFoundError) as context:
self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={},
)
self.assertIn("[Errno 2] No such file or directory: 'blah'",
format(context.exception))
def test_exceptions_in_responder_are_handled_and_sent_to_asker(self):
"""Test that exceptions raised in the child service are handled and sent back to the asker."""
child = self.make_child_service_with_error(
twined.exceptions.InvalidManifestContents(
"'met_mast_id' is a required property"))
parent = MockService(backend=BACKEND, children={child.id: child})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
with self.assertRaises(twined.exceptions.
InvalidManifestContents) as context:
self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={},
)
self.assertIn("'met_mast_id' is a required property",
context.exception.args[0])
def test_emit_with_non_json_serialisable_args(self):
"""Test that non-JSON-serialisable arguments to log messages are converted to their string representation
before being serialised and published to the Pub/Sub topic.
"""
backend = GCPPubSubBackend(project_name="blah")
service = MockService(backend=backend)
topic = MockTopic(name="world-1", namespace="hello", service=service)
topic.create()
non_json_serialisable_thing = NonJSONSerialisable()
# Check that it can't be serialised to JSON.
with self.assertRaises(TypeError):
json.dumps(non_json_serialisable_thing)
record = logging.makeLogRecord({
"msg": "%r is not JSON-serialisable but can go into a log message",
"args": (non_json_serialisable_thing, )
})
with patch("tests.cloud.pub_sub.mocks.MockPublisher.publish"
) as mock_publish:
GooglePubSubHandler(service.publisher, topic,
"analysis-id").emit(record)
self.assertEqual(
json.loads(mock_publish.call_args.kwargs["data"].decode())
["log_record"]["msg"],
"NonJSONSerialisableInstance is not JSON-serialisable but can go into a log message",
)
def test_service_can_ask_multiple_questions_to_child(self):
"""Test that a service can ask multiple questions to the same child and expect replies to them all."""
child = self.make_new_child(BACKEND,
run_function_returnee=MockAnalysis(),
use_mock=True)
parent = MockService(backend=BACKEND, children={child.id: child})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
answers = []
for i in range(5):
answers.append(
self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={},
))
for answer in answers:
self.assertEqual(
answer,
{
"output_values": MockAnalysis().output_values,
"output_manifest": MockAnalysis().output_manifest
},
)
def test_ask_on_non_existent_service_results_in_error(self):
"""Test that trying to ask a question to a non-existent service (i.e. one without a topic in Google Pub/Sub)
results in an error.
"""
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with self.assertRaises(exceptions.ServiceNotFound):
MockService(backend=BACKEND).ask(service_id="hello",
input_values=[1, 2, 3, 4])
def test_ask_with_input_manifest_with_local_paths_works_if_allowed_and_child_has_access_to_the_local_paths(
self):
"""Test that an input manifest referencing local files can be used if the files can be accessed by the child and
the `allow_local_files` parameter is `True`.
"""
temporary_local_path = tempfile.NamedTemporaryFile(delete=False).name
with open(temporary_local_path, "w") as f:
f.write("This is a local file.")
local_file = Datafile(path=temporary_local_path)
self.assertFalse(local_file.exists_in_cloud)
manifest = Manifest(
datasets={
"my-local-dataset":
Dataset(name="my-local-dataset", files={local_file})
})
# Get the child to open the local file itself and return the contents as output.
def run_function(analysis_id, input_values, input_manifest,
analysis_log_handler, handle_monitor_message):
with open(temporary_local_path) as f:
return MockAnalysis(output_values=f.read())
child = MockService(backend=BACKEND, run_function=run_function)
parent = MockService(backend=BACKEND, children={child.id: child})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
answer = self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={},
input_manifest=manifest,
allow_local_files=True,
)
self.assertEqual(answer["output_values"], "This is a local file.")
def test_ask_with_input_manifest_with_local_paths_raises_error(self):
"""Test that an error is raised if an input manifest whose datasets and/or files are not located in the cloud
is used in a question.
"""
with self.assertRaises(exceptions.FileLocationError):
MockService(backend=BACKEND).ask(
service_id=str(uuid.uuid4()),
input_values={},
input_manifest=self.create_valid_manifest(),
)
def test_with_monitor_message_handler(self):
"""Test that monitor messages can be sent from a child app and handled by the parent's monitor message handler."""
def create_run_function_with_monitoring():
def mock_app(analysis):
analysis.send_monitor_message(
{"status": "my first monitor message"})
analysis.send_monitor_message(
{"status": "my second monitor message"})
twine = """
{
"input_values_schema": {"type": "object", "required": []},
"monitor_message_schema": {
"type": "object",
"properties": {"status": {"type": "string"}},
"required": ["status"]
}
}
"""
return Runner(app_src=mock_app, twine=twine).run
child = MockService(backend=BACKEND,
run_function=create_run_function_with_monitoring())
parent = MockService(backend=BACKEND, children={child.id: child})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
subscription, _ = parent.ask(child.id, input_values={})
monitoring_data = []
parent.wait_for_answer(subscription,
handle_monitor_message=lambda data:
monitoring_data.append(data))
self.assertEqual(monitoring_data,
[{
"status": "my first monitor message"
}, {
"status": "my second monitor message"
}])
def test_service_can_ask_questions_to_multiple_children(self):
"""Test that a service can ask questions to different children and expect replies to them all."""
child_1 = self.make_new_child(BACKEND,
run_function_returnee=MockAnalysis(),
use_mock=True)
child_2 = self.make_new_child(
BACKEND,
run_function_returnee=DifferentMockAnalysis(),
use_mock=True)
parent = MockService(backend=BACKEND,
children={
child_1.id: child_1,
child_2.id: child_2
})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child_1.serve()
child_2.serve()
answer_1 = self.ask_question_and_wait_for_answer(
parent=parent,
child=child_1,
input_values={},
)
answer_2 = self.ask_question_and_wait_for_answer(
parent=parent,
child=child_2,
input_values={},
)
self.assertEqual(
answer_1,
{
"output_values": MockAnalysis().output_values,
"output_manifest": MockAnalysis().output_manifest
},
)
self.assertEqual(
answer_2,
{
"output_values": DifferentMockAnalysis.output_values,
"output_manifest": DifferentMockAnalysis.output_manifest,
},
)
def test_child_can_be_asked_multiple_questions(self):
"""Test that a child can be asked multiple questions."""
backend = GCPPubSubBackend(project_name="blah")
def run_function(analysis_id, input_values, input_manifest,
analysis_log_handler, handle_monitor_message):
return MockAnalysis(output_values=input_values)
responding_service = MockService(backend=backend,
service_id="testing/wind-speed",
run_function=run_function)
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("octue.resources.child.BACKEND_TO_SERVICE_MAPPING",
{"GCPPubSubBackend": MockService}):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
responding_service.serve()
child = Child(
id=responding_service.id,
backend={
"name": "GCPPubSubBackend",
"project_name": "blah"
},
)
# Make sure the child's underlying mock service knows how to access the mock responding service.
child._service.children[
responding_service.id] = responding_service
self.assertEqual(
child.ask([1, 2, 3, 4])["output_values"],
[1, 2, 3, 4])
self.assertEqual(
child.ask([5, 6, 7, 8])["output_values"],
[5, 6, 7, 8])
def setUpClass(cls):
"""Set up the test class with a mock subscription.
:return None:
"""
service = MockService(backend=BACKEND)
mock_topic = MockTopic(name="world", namespace="hello", service=service)
cls.mock_subscription = MockSubscription(
name="world",
topic=mock_topic,
namespace="hello",
project_name=TEST_PROJECT_NAME,
subscriber=MockSubscriber(),
)
def test_emit(self):
"""Test the log message is published when `GooglePubSubHandler.emit` is called."""
backend = GCPPubSubBackend(project_name="blah")
service = MockService(backend=backend)
topic = MockTopic(name="world", namespace="hello", service=service)
topic.create()
log_record = makeLogRecord({"msg": "Starting analysis."})
GooglePubSubHandler(service.publisher, topic,
"analysis-id").emit(log_record)
self.assertEqual(
json.loads(
MESSAGES[topic.name][0].data.decode())["log_record"]["msg"],
"Starting analysis.")
def test_ask_with_input_manifest(self):
"""Test that a service can ask a question including an input manifest to another service that is serving and
receive an answer.
"""
child = self.make_new_child(BACKEND,
run_function_returnee=MockAnalysis(),
use_mock=True)
parent = MockService(backend=BACKEND, children={child.id: child})
dataset_path = f"gs://{TEST_BUCKET_NAME}/my-dataset"
input_manifest = Manifest(
datasets={
"my-dataset":
Dataset(
files=[
f"{dataset_path}/hello.txt",
f"{dataset_path}/goodbye.csv"
],
path=dataset_path,
)
})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
with patch(
"google.cloud.storage.blob.Blob.generate_signed_url",
new=mock_generate_signed_url):
answer = self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={},
input_manifest=input_manifest,
)
self.assertEqual(
answer,
{
"output_values": MockAnalysis().output_values,
"output_manifest": MockAnalysis().output_manifest
},
)
def make_new_child(backend, run_function_returnee, use_mock=False):
"""Make and return a new child service that returns the given run function returnee when its run function is
executed.
:param octue.resources.service_backends.ServiceBackend backend:
:param any run_function_returnee:
:param bool use_mock:
:return tests.cloud.pub_sub.mocks.MockService:
"""
run_function = (lambda analysis_id, input_values, input_manifest,
analysis_log_handler, handle_monitor_message:
run_function_returnee)
if use_mock:
return MockService(backend=backend, run_function=run_function)
return Service(backend=backend, run_function=run_function)
def test_warning_issued_if_child_and_parent_sdk_versions_incompatible(
self):
"""Test that a warning is logged if the parent and child's Octue SDK versions are potentially incompatible."""
child = self.make_new_child(backend=BACKEND,
run_function_returnee=MockAnalysis(),
use_mock=True)
parent = MockService(backend=BACKEND, children={child.id: child})
with patch("octue.cloud.pub_sub.service.Topic", new=MockTopic):
with patch("octue.cloud.pub_sub.service.Subscription",
new=MockSubscription):
with patch("google.cloud.pubsub_v1.SubscriberClient",
new=MockSubscriber):
child.serve()
for parent_sdk_version, child_sdk_version in (
("0.1.0", "0.2.0"),
("0.2.0", "0.1.0"),
("0.1.0", "1.1.0"),
("1.1.0", "0.1.0"),
):
with self.subTest(
parent_sdk_version=parent_sdk_version,
child_sdk_version=child_sdk_version):
with patch("pkg_resources.Distribution.version",
child_sdk_version):
with self.assertLogs() as logging_context:
self.ask_question_and_wait_for_answer(
parent=parent,
child=child,
input_values={
"question":
"What does the child of the child say?"
},
parent_sdk_version=parent_sdk_version,
)
self.assertIn(
f"The parent's Octue SDK version {parent_sdk_version} may not be compatible "
f"with the local Octue SDK version {child_sdk_version}",
logging_context.output[3],
)