def test_two_stubs_with_parallel_calls_and_job_response_errors_raised(
self, stub_test_action_1, stub_test_action_2):
stub_test_action_1.return_value = JobResponse(
errors=[Error(code='BAD_JOB', message='You are a bad job')])
with self.assertRaises(self.client.JobError) as error_context:
self.client.call_jobs_parallel([
{
'service_name': 'test_service',
'actions': [{
'action': 'test_action_1',
'body': {
'a': 'b'
}
}]
},
{
'service_name': 'test_service',
'actions': [{
'action': 'test_action_2',
'body': {
'c': 'd'
}
}]
},
], )
self.assertEqual([Error(code='BAD_JOB', message='You are a bad job')],
error_context.exception.errors)
stub_test_action_1.assert_called_once_with({'a': 'b'})
stub_test_action_2.assert_called_once_with({'c': 'd'})
def handle_error(self, error, variables=None):
"""
Makes a last-ditch error response
"""
# Get the error and traceback if we can
try:
error_str, traceback_str = str(error), traceback.format_exc()
except Exception:
self.metrics.counter(
'server.error.error_formatting_failure').increment()
error_str, traceback_str = 'Error formatting error', traceback.format_exc(
)
# Log what happened
self.logger.exception(error)
# Make a bare bones job response
error_dict = {
'code': ERROR_CODE_SERVER_ERROR,
'message': 'Internal server error: %s' % error_str,
'traceback': traceback_str,
}
if variables is not None:
try:
error_dict['variables'] = {
key: repr(value)
for key, value in variables.items()
}
except Exception:
self.metrics.counter(
'server.error.variable_formatting_failure').increment()
error_dict['variables'] = 'Error formatting variables'
job_response = JobResponse(errors=[error_dict])
return job_response
def _get_response(self):
with self.metrics.timer('client.receive.excluding_middleware'):
request_id, meta, message = self.transport.receive_response_message()
if message is None:
return None, None
else:
return request_id, JobResponse(**message)
def test_check_client_settings_with_settings(self):
client = Client({
'foo': {'transport': {'path': 'pysoa.test.stub_service:StubClientTransport'}},
'bar': {'transport': {'path': 'pysoa.test.stub_service:StubClientTransport'}},
'baz': {'transport': {'path': 'pysoa.test.stub_service:StubClientTransport'}},
'qux': {'transport': {'path': 'pysoa.test.stub_service:StubClientTransport'}},
})
action_request = EnrichedActionRequest(action='status', body={}, switches=None, client=client)
baz_body = {
'conformity': '1.2.3',
'pysoa': '1.0.2',
'python': '3.7.4',
'version': '9.7.8',
}
with stub_action('foo', 'status') as foo_stub,\
stub_action('bar', 'status', errors=[Error('BAR_ERROR', 'Bar error')]),\
stub_action('baz', 'status', body=baz_body),\
stub_action('qux', 'status') as qux_stub:
foo_stub.return_value = JobResponse(errors=[Error('FOO_ERROR', 'Foo error')])
qux_stub.side_effect = MessageReceiveTimeout('Timeout calling qux')
response = _CheckOtherServicesAction()(action_request)
self.assertIsInstance(response, ActionResponse)
self.assertEqual(six.text_type(conformity.__version__), response.body['conformity'])
self.assertEqual(six.text_type(pysoa.__version__), response.body['pysoa'])
self.assertEqual(six.text_type(platform.python_version()), response.body['python'])
self.assertEqual('8.71.2', response.body['version'])
self.assertIn('healthcheck', response.body)
self.assertEqual([], response.body['healthcheck']['warnings'])
self.assertIn(
('FOO_CALL_ERROR', six.text_type([Error('FOO_ERROR', 'Foo error')])),
response.body['healthcheck']['errors'],
)
self.assertIn(
('BAR_STATUS_ERROR', six.text_type([Error('BAR_ERROR', 'Bar error')])),
response.body['healthcheck']['errors'],
)
self.assertIn(
('QUX_TRANSPORT_ERROR', 'Timeout calling qux'),
response.body['healthcheck']['errors'],
)
self.assertEqual(3, len(response.body['healthcheck']['errors']))
self.assertEqual(
{
'services': {
'baz': {
'conformity': '1.2.3',
'pysoa': '1.0.2',
'python': '3.7.4',
'version': '9.7.8',
},
},
},
response.body['healthcheck']['diagnostics'],
)
def execute_job(self, job_request):
"""
Processes and runs the ActionRequests on the Job.
"""
# Run the Job's Actions
job_response = JobResponse()
job_switches = RequestSwitchSet(job_request['context']['switches'])
for i, raw_action_request in enumerate(job_request['actions']):
action_request = EnrichedActionRequest(
action=raw_action_request['action'],
body=raw_action_request.get('body', None),
switches=job_switches,
context=job_request['context'],
control=job_request['control'],
client=job_request['client'],
)
if action_request.action in self.action_class_map:
# Get action to run
action = self.action_class_map[action_request.action](
self.settings)
# Wrap it in middleware
wrapper = self.make_middleware_stack(
[m.action for m in self.middleware],
action,
)
# Execute the middleware stack
try:
action_response = wrapper(action_request)
except ActionError as e:
# Error: an error was thrown while running the Action (or Action middleware)
action_response = ActionResponse(
action=action_request.action,
errors=e.errors,
)
else:
# Error: Action not found.
action_response = ActionResponse(
action=action_request.action,
errors=[
Error(
code=ERROR_CODE_UNKNOWN,
message=
'The action "{}" was not found on this server.'.
format(action_request.action),
field='action',
)
],
)
job_response.actions.append(action_response)
if (action_response.errors and not job_request['control'].get(
'continue_on_error', False)):
# Quit running Actions if an error occurred and continue_on_error is False
break
return job_response
def process_job(self, job_request):
"""
Validate, execute, and run the job request, wrapping it with any applicable job middleware.
:param job_request: The job request
:type job_request: dict
:return: A `JobResponse` object
:rtype: JobResponse
:raise: JobError
"""
try:
# Validate JobRequest message
validation_errors = [
Error(
code=error.code,
message=error.message,
field=error.pointer,
) for error in (JobRequestSchema.errors(job_request) or [])
]
if validation_errors:
raise JobError(errors=validation_errors)
# Add the client object in case a middleware wishes to use it
job_request['client'] = self.make_client(job_request['context'])
# Add the async event loop in case a middleware wishes to use it
job_request['async_event_loop'] = self._async_event_loop
if self._async_event_loop_thread:
job_request[
'run_coroutine'] = self._async_event_loop_thread.run_coroutine
else:
job_request['run_coroutine'] = None
# Build set of middleware + job handler, then run job
wrapper = self.make_middleware_stack(
[m.job for m in self.middleware],
self.execute_job,
)
job_response = wrapper(job_request)
if 'correlation_id' in job_request['context']:
job_response.context['correlation_id'] = job_request[
'context']['correlation_id']
except JobError as e:
self.metrics.counter('server.error.job_error').increment()
job_response = JobResponse(errors=e.errors, )
except Exception as e:
# Send an error response if no middleware caught this.
# Formatting the error might itself error, so try to catch that
self.metrics.counter('server.error.unhandled_error').increment()
return self.handle_job_exception(e)
return job_response
def handle_job_error_code(self, code, message, request_for_logging, response_for_logging, extra=None):
log_extra = {'data': {'request': request_for_logging, 'response': response_for_logging}}
if extra:
log_extra['data'].update(extra)
self.logger.error(
message,
exc_info=True,
extra=log_extra,
)
return JobResponse(errors=[Error(code=code, message=message)])
def test_mock_action_with_job_error_response_raises_job_error_exception(self, stub_test_action_2):
stub_test_action_2.return_value = JobResponse(errors=[Error(code='BAR_BAD_JOB', message='Uh-uh job')])
with self.assertRaises(Client.JobError) as e:
self.client.call_action('test_service', 'test_action_2', {'a': 'body'})
self.assertEqual('BAR_BAD_JOB', e.exception.errors[0].code)
self.assertIsNone(e.exception.errors[0].field)
self.assertEqual('Uh-uh job', e.exception.errors[0].message)
self.assertEqual(1, stub_test_action_2.call_count)
self.assertEqual({'a': 'body'}, stub_test_action_2.call_body)
stub_test_action_2.assert_called_once_with({'a': 'body'})
def handle_job_exception(self, exception, variables=None):
"""
Makes and returns a last-ditch error response.
:param exception: The exception that happened
:type exception: Exception
:param variables: A dictionary of context-relevant variables to include in the error response
:type variables: dict
:return: A `JobResponse` object
:rtype: JobResponse
"""
# Get the error and traceback if we can
# noinspection PyBroadException
try:
error_str, traceback_str = six.text_type(
exception), traceback.format_exc()
except Exception:
self.metrics.counter(
'server.error.error_formatting_failure').increment()
error_str, traceback_str = 'Error formatting error', traceback.format_exc(
)
# Log what happened
self.logger.exception(exception)
if not isinstance(traceback_str, six.text_type):
try:
# Try to
traceback_str = traceback_str.decode('utf-8')
except UnicodeDecodeError:
traceback_str = 'UnicodeDecodeError: Traceback could not be decoded'
# Make a bare bones job response
error_dict = {
'code': ERROR_CODE_SERVER_ERROR,
'message': 'Internal server error: %s' % error_str,
'traceback': traceback_str,
}
if variables is not None:
# noinspection PyBroadException
try:
error_dict['variables'] = {
key: repr(value)
for key, value in variables.items()
}
except Exception:
self.metrics.counter(
'server.error.variable_formatting_failure').increment()
error_dict['variables'] = 'Error formatting variables'
return JobResponse(errors=[error_dict])
def process_job(self, job_request):
"""
Validate, execute, and run Job-level middleware for JobRequests.
Args:
job_request: a JobRequest dictionary.
Returns:
A JobResponse instance.
"""
try:
# Validate JobRequest message
validation_errors = [
Error(
code=ERROR_CODE_INVALID,
message=error.message,
field=error.pointer,
) for error in (JobRequestSchema.errors(job_request) or [])
]
if validation_errors:
raise JobError(errors=validation_errors)
# Add a client router in case a middleware wishes to use it
job_request['client'] = self.make_client(job_request['context'])
# Build set of middleware + job handler, then run job
wrapper = self.make_middleware_stack(
[m.job for m in self.middleware],
self.execute_job,
)
job_response = wrapper(job_request)
except JobError as e:
self.metrics.counter('server.error.job_error').increment()
job_response = JobResponse(errors=e.errors, )
except Exception as e:
# Send an error response if no middleware caught this.
# Formatting the error might itself error, so try to catch that
self.metrics.counter('server.error.unhandled_error').increment()
return self.handle_error(e)
return job_response
def execute_job(self, job_request):
"""
Processes and runs the action requests contained in the job and returns a `JobResponse`.
:param job_request: The job request
:type job_request: dict
:return: A `JobResponse` object
:rtype: JobResponse
"""
# Run the Job's Actions
job_response = JobResponse()
job_switches = RequestSwitchSet(job_request['context']['switches'])
for i, raw_action_request in enumerate(job_request['actions']):
action_request = EnrichedActionRequest(
action=raw_action_request['action'],
body=raw_action_request.get('body', None),
switches=job_switches,
context=job_request['context'],
control=job_request['control'],
client=job_request['client'],
async_event_loop=job_request['async_event_loop'],
)
action_in_class_map = action_request.action in self.action_class_map
if action_in_class_map or action_request.action in ('status', 'introspect'):
# Get action to run
if action_in_class_map:
action = self.action_class_map[action_request.action](self.settings)
elif action_request.action == 'introspect':
from pysoa.server.action.introspection import IntrospectionAction
action = IntrospectionAction(server=self)
else:
if not self._default_status_action_class:
from pysoa.server.action.status import make_default_status_action_class
self._default_status_action_class = make_default_status_action_class(self.__class__)
action = self._default_status_action_class(self.settings)
# Wrap it in middleware
wrapper = self.make_middleware_stack(
[m.action for m in self.middleware],
action,
)
# Execute the middleware stack
try:
action_response = wrapper(action_request)
except ActionError as e:
# Error: an error was thrown while running the Action (or Action middleware)
action_response = ActionResponse(
action=action_request.action,
errors=e.errors,
)
else:
# Error: Action not found.
action_response = ActionResponse(
action=action_request.action,
errors=[Error(
code=ERROR_CODE_UNKNOWN,
message='The action "{}" was not found on this server.'.format(action_request.action),
field='action',
)],
)
job_response.actions.append(action_response)
if (
action_response.errors and
not job_request['control'].get('continue_on_error', False)
):
# Quit running Actions if an error occurred and continue_on_error is False
break
return job_response
def wrapped_send_request(client, service_name, actions, *args,
**kwargs):
assert isinstance(
service_name, six.text_type
), 'Called service name "{}" must be unicode'.format(
service_name, )
actions_to_send_to_mock = OrderedDict()
actions_to_send_to_wrapped_client = []
for i, action_request in enumerate(actions):
action_name = getattr(action_request, 'action',
None) or action_request['action']
assert isinstance(
action_name, six.text_type
), 'Called action name "{}" must be unicode'.format(
action_name, )
if service_name == self.service and action_name == self.action:
# If the service AND action name match, we should send the request to our mocked client
if not isinstance(action_request, ActionRequest):
action_request = ActionRequest(**action_request)
actions_to_send_to_mock[i] = action_request
else:
# If the service OR action name DO NOT match, we should delegate the request to the wrapped client
actions_to_send_to_wrapped_client.append(action_request)
request_id = _global_stub_action_request_counter.get_next()
continue_on_error = kwargs.get('continue_on_error', False)
if actions_to_send_to_wrapped_client:
# If any un-stubbed actions need to be sent to the original client, send them
self._services_with_calls_sent_to_wrapped_client.add(
service_name)
unwrapped_request_id = self._wrapped_client_send_request(
client, service_name, actions_to_send_to_wrapped_client,
*args, **kwargs)
if not actions_to_send_to_mock:
# If there are no stubbed actions to mock, just return the un-stubbed request ID
return unwrapped_request_id
self._stub_action_responses_to_merge[service_name][
unwrapped_request_id] = (
request_id,
continue_on_error,
)
ordered_actions_for_merging = OrderedDict()
job_response_transport_exception = None
job_response = JobResponse()
for i, action_request in actions_to_send_to_mock.items():
mock_response = None
try:
mock_response = mock_action(action_request.body or {})
if isinstance(mock_response, JobResponse):
job_response.errors.extend(mock_response.errors)
if mock_response.actions:
mock_response = mock_response.actions[0]
elif isinstance(mock_response, dict):
mock_response = ActionResponse(self.action,
body=mock_response)
elif not isinstance(mock_response, ActionResponse):
mock_response = ActionResponse(self.action)
except ActionError as e:
mock_response = ActionResponse(self.action,
errors=e.errors)
except JobError as e:
job_response.errors.extend(e.errors)
except (MessageReceiveError, MessageReceiveTimeout) as e:
job_response_transport_exception = e
if mock_response:
ordered_actions_for_merging[i] = mock_response
job_response.actions.append(mock_response)
if not continue_on_error and mock_response.errors:
break
if job_response.errors:
break
if actions_to_send_to_wrapped_client:
# If the responses will have to be merged by get_all_responses, replace the list with the ordered dict
job_response.actions = ordered_actions_for_merging
self._stub_action_responses_outstanding[service_name][
request_id] = (job_response_transport_exception
or job_response)
return request_id
def wrapped(client, service_name, actions, *args, **kwargs):
assert isinstance(
service_name, six.text_type
), 'Called service name "{}" must be unicode'.format(
service_name, )
requests_to_send_to_mock_client = OrderedDict()
requests_to_send_to_wrapped_client = []
for i, action_request in enumerate(actions):
action_name = getattr(action_request, 'action',
None) or action_request['action']
assert isinstance(
action_name, six.text_type
), 'Called action name "{}" must be unicode'.format(
action_name, )
if service_name == self.service and action_name == self.action:
# If the service AND action name match, we should send the request to our mocked client
if not isinstance(action_request, ActionRequest):
action_request = ActionRequest(action_request)
requests_to_send_to_mock_client[i] = action_request
else:
# If the service OR action name DO NOT match, we should delegate the request to the wrapped client
requests_to_send_to_wrapped_client.append(action_request)
# Hold off on raising action errors until both mock and real responses are merged
raise_action_errors = kwargs.get('raise_action_errors', True)
kwargs['raise_action_errors'] = False
# Run the real and mocked jobs and merge the results, to simulate a single job
if requests_to_send_to_wrapped_client:
job_response = self._wrapped_client_call_actions_method(
client, service_name, requests_to_send_to_wrapped_client,
*args, **kwargs)
else:
job_response = JobResponse()
for i, action_request in requests_to_send_to_mock_client.items():
try:
mock_response = mock_action(action_request.body or {})
if isinstance(mock_response, JobResponse):
mock_response = mock_response.actions[0]
elif isinstance(mock_response, dict):
mock_response = ActionResponse(self.action,
body=mock_response)
elif not isinstance(mock_response, ActionResponse):
mock_response = ActionResponse(self.action)
except ActionError as e:
mock_response = ActionResponse(self.action,
errors=e.errors)
job_response.actions.insert(i, mock_response)
if kwargs.get('continue_on_error', False) is False:
# Simulate the server job halting on the first action error
first_error_index = -1
for i, action_result in enumerate(job_response.actions):
if action_result.errors:
first_error_index = i
break
if first_error_index >= 0:
job_response.actions = job_response.actions[:
first_error_index
+ 1]
if raise_action_errors:
error_actions = [
action for action in job_response.actions if action.errors
]
if error_actions:
raise Client.CallActionError(error_actions)
return job_response