def test_two_stubs_with_parallel_calls_and_job_response_errors_raised( self, stub_test_action_1, stub_test_action_2): stub_test_action_1.return_value = JobResponse( errors=[Error(code='BAD_JOB', message='You are a bad job')]) with self.assertRaises(self.client.JobError) as error_context: self.client.call_jobs_parallel([ { 'service_name': 'test_service', 'actions': [{ 'action': 'test_action_1', 'body': { 'a': 'b' } }] }, { 'service_name': 'test_service', 'actions': [{ 'action': 'test_action_2', 'body': { 'c': 'd' } }] }, ], ) self.assertEqual([Error(code='BAD_JOB', message='You are a bad job')], error_context.exception.errors) stub_test_action_1.assert_called_once_with({'a': 'b'}) stub_test_action_2.assert_called_once_with({'c': 'd'})
def handle_error(self, error, variables=None): """ Makes a last-ditch error response """ # Get the error and traceback if we can try: error_str, traceback_str = str(error), traceback.format_exc() except Exception: self.metrics.counter( 'server.error.error_formatting_failure').increment() error_str, traceback_str = 'Error formatting error', traceback.format_exc( ) # Log what happened self.logger.exception(error) # Make a bare bones job response error_dict = { 'code': ERROR_CODE_SERVER_ERROR, 'message': 'Internal server error: %s' % error_str, 'traceback': traceback_str, } if variables is not None: try: error_dict['variables'] = { key: repr(value) for key, value in variables.items() } except Exception: self.metrics.counter( 'server.error.variable_formatting_failure').increment() error_dict['variables'] = 'Error formatting variables' job_response = JobResponse(errors=[error_dict]) return job_response
def _get_response(self): with self.metrics.timer('client.receive.excluding_middleware'): request_id, meta, message = self.transport.receive_response_message() if message is None: return None, None else: return request_id, JobResponse(**message)
def test_check_client_settings_with_settings(self): client = Client({ 'foo': {'transport': {'path': 'pysoa.test.stub_service:StubClientTransport'}}, 'bar': {'transport': {'path': 'pysoa.test.stub_service:StubClientTransport'}}, 'baz': {'transport': {'path': 'pysoa.test.stub_service:StubClientTransport'}}, 'qux': {'transport': {'path': 'pysoa.test.stub_service:StubClientTransport'}}, }) action_request = EnrichedActionRequest(action='status', body={}, switches=None, client=client) baz_body = { 'conformity': '1.2.3', 'pysoa': '1.0.2', 'python': '3.7.4', 'version': '9.7.8', } with stub_action('foo', 'status') as foo_stub,\ stub_action('bar', 'status', errors=[Error('BAR_ERROR', 'Bar error')]),\ stub_action('baz', 'status', body=baz_body),\ stub_action('qux', 'status') as qux_stub: foo_stub.return_value = JobResponse(errors=[Error('FOO_ERROR', 'Foo error')]) qux_stub.side_effect = MessageReceiveTimeout('Timeout calling qux') response = _CheckOtherServicesAction()(action_request) self.assertIsInstance(response, ActionResponse) self.assertEqual(six.text_type(conformity.__version__), response.body['conformity']) self.assertEqual(six.text_type(pysoa.__version__), response.body['pysoa']) self.assertEqual(six.text_type(platform.python_version()), response.body['python']) self.assertEqual('8.71.2', response.body['version']) self.assertIn('healthcheck', response.body) self.assertEqual([], response.body['healthcheck']['warnings']) self.assertIn( ('FOO_CALL_ERROR', six.text_type([Error('FOO_ERROR', 'Foo error')])), response.body['healthcheck']['errors'], ) self.assertIn( ('BAR_STATUS_ERROR', six.text_type([Error('BAR_ERROR', 'Bar error')])), response.body['healthcheck']['errors'], ) self.assertIn( ('QUX_TRANSPORT_ERROR', 'Timeout calling qux'), response.body['healthcheck']['errors'], ) self.assertEqual(3, len(response.body['healthcheck']['errors'])) self.assertEqual( { 'services': { 'baz': { 'conformity': '1.2.3', 'pysoa': '1.0.2', 'python': '3.7.4', 'version': '9.7.8', }, }, }, response.body['healthcheck']['diagnostics'], )
def execute_job(self, job_request): """ Processes and runs the ActionRequests on the Job. """ # Run the Job's Actions job_response = JobResponse() job_switches = RequestSwitchSet(job_request['context']['switches']) for i, raw_action_request in enumerate(job_request['actions']): action_request = EnrichedActionRequest( action=raw_action_request['action'], body=raw_action_request.get('body', None), switches=job_switches, context=job_request['context'], control=job_request['control'], client=job_request['client'], ) if action_request.action in self.action_class_map: # Get action to run action = self.action_class_map[action_request.action]( self.settings) # Wrap it in middleware wrapper = self.make_middleware_stack( [m.action for m in self.middleware], action, ) # Execute the middleware stack try: action_response = wrapper(action_request) except ActionError as e: # Error: an error was thrown while running the Action (or Action middleware) action_response = ActionResponse( action=action_request.action, errors=e.errors, ) else: # Error: Action not found. action_response = ActionResponse( action=action_request.action, errors=[ Error( code=ERROR_CODE_UNKNOWN, message= 'The action "{}" was not found on this server.'. format(action_request.action), field='action', ) ], ) job_response.actions.append(action_response) if (action_response.errors and not job_request['control'].get( 'continue_on_error', False)): # Quit running Actions if an error occurred and continue_on_error is False break return job_response
def process_job(self, job_request): """ Validate, execute, and run the job request, wrapping it with any applicable job middleware. :param job_request: The job request :type job_request: dict :return: A `JobResponse` object :rtype: JobResponse :raise: JobError """ try: # Validate JobRequest message validation_errors = [ Error( code=error.code, message=error.message, field=error.pointer, ) for error in (JobRequestSchema.errors(job_request) or []) ] if validation_errors: raise JobError(errors=validation_errors) # Add the client object in case a middleware wishes to use it job_request['client'] = self.make_client(job_request['context']) # Add the async event loop in case a middleware wishes to use it job_request['async_event_loop'] = self._async_event_loop if self._async_event_loop_thread: job_request[ 'run_coroutine'] = self._async_event_loop_thread.run_coroutine else: job_request['run_coroutine'] = None # Build set of middleware + job handler, then run job wrapper = self.make_middleware_stack( [m.job for m in self.middleware], self.execute_job, ) job_response = wrapper(job_request) if 'correlation_id' in job_request['context']: job_response.context['correlation_id'] = job_request[ 'context']['correlation_id'] except JobError as e: self.metrics.counter('server.error.job_error').increment() job_response = JobResponse(errors=e.errors, ) except Exception as e: # Send an error response if no middleware caught this. # Formatting the error might itself error, so try to catch that self.metrics.counter('server.error.unhandled_error').increment() return self.handle_job_exception(e) return job_response
def handle_job_error_code(self, code, message, request_for_logging, response_for_logging, extra=None): log_extra = {'data': {'request': request_for_logging, 'response': response_for_logging}} if extra: log_extra['data'].update(extra) self.logger.error( message, exc_info=True, extra=log_extra, ) return JobResponse(errors=[Error(code=code, message=message)])
def test_mock_action_with_job_error_response_raises_job_error_exception(self, stub_test_action_2): stub_test_action_2.return_value = JobResponse(errors=[Error(code='BAR_BAD_JOB', message='Uh-uh job')]) with self.assertRaises(Client.JobError) as e: self.client.call_action('test_service', 'test_action_2', {'a': 'body'}) self.assertEqual('BAR_BAD_JOB', e.exception.errors[0].code) self.assertIsNone(e.exception.errors[0].field) self.assertEqual('Uh-uh job', e.exception.errors[0].message) self.assertEqual(1, stub_test_action_2.call_count) self.assertEqual({'a': 'body'}, stub_test_action_2.call_body) stub_test_action_2.assert_called_once_with({'a': 'body'})
def handle_job_exception(self, exception, variables=None): """ Makes and returns a last-ditch error response. :param exception: The exception that happened :type exception: Exception :param variables: A dictionary of context-relevant variables to include in the error response :type variables: dict :return: A `JobResponse` object :rtype: JobResponse """ # Get the error and traceback if we can # noinspection PyBroadException try: error_str, traceback_str = six.text_type( exception), traceback.format_exc() except Exception: self.metrics.counter( 'server.error.error_formatting_failure').increment() error_str, traceback_str = 'Error formatting error', traceback.format_exc( ) # Log what happened self.logger.exception(exception) if not isinstance(traceback_str, six.text_type): try: # Try to traceback_str = traceback_str.decode('utf-8') except UnicodeDecodeError: traceback_str = 'UnicodeDecodeError: Traceback could not be decoded' # Make a bare bones job response error_dict = { 'code': ERROR_CODE_SERVER_ERROR, 'message': 'Internal server error: %s' % error_str, 'traceback': traceback_str, } if variables is not None: # noinspection PyBroadException try: error_dict['variables'] = { key: repr(value) for key, value in variables.items() } except Exception: self.metrics.counter( 'server.error.variable_formatting_failure').increment() error_dict['variables'] = 'Error formatting variables' return JobResponse(errors=[error_dict])
def process_job(self, job_request): """ Validate, execute, and run Job-level middleware for JobRequests. Args: job_request: a JobRequest dictionary. Returns: A JobResponse instance. """ try: # Validate JobRequest message validation_errors = [ Error( code=ERROR_CODE_INVALID, message=error.message, field=error.pointer, ) for error in (JobRequestSchema.errors(job_request) or []) ] if validation_errors: raise JobError(errors=validation_errors) # Add a client router in case a middleware wishes to use it job_request['client'] = self.make_client(job_request['context']) # Build set of middleware + job handler, then run job wrapper = self.make_middleware_stack( [m.job for m in self.middleware], self.execute_job, ) job_response = wrapper(job_request) except JobError as e: self.metrics.counter('server.error.job_error').increment() job_response = JobResponse(errors=e.errors, ) except Exception as e: # Send an error response if no middleware caught this. # Formatting the error might itself error, so try to catch that self.metrics.counter('server.error.unhandled_error').increment() return self.handle_error(e) return job_response
def execute_job(self, job_request): """ Processes and runs the action requests contained in the job and returns a `JobResponse`. :param job_request: The job request :type job_request: dict :return: A `JobResponse` object :rtype: JobResponse """ # Run the Job's Actions job_response = JobResponse() job_switches = RequestSwitchSet(job_request['context']['switches']) for i, raw_action_request in enumerate(job_request['actions']): action_request = EnrichedActionRequest( action=raw_action_request['action'], body=raw_action_request.get('body', None), switches=job_switches, context=job_request['context'], control=job_request['control'], client=job_request['client'], async_event_loop=job_request['async_event_loop'], ) action_in_class_map = action_request.action in self.action_class_map if action_in_class_map or action_request.action in ('status', 'introspect'): # Get action to run if action_in_class_map: action = self.action_class_map[action_request.action](self.settings) elif action_request.action == 'introspect': from pysoa.server.action.introspection import IntrospectionAction action = IntrospectionAction(server=self) else: if not self._default_status_action_class: from pysoa.server.action.status import make_default_status_action_class self._default_status_action_class = make_default_status_action_class(self.__class__) action = self._default_status_action_class(self.settings) # Wrap it in middleware wrapper = self.make_middleware_stack( [m.action for m in self.middleware], action, ) # Execute the middleware stack try: action_response = wrapper(action_request) except ActionError as e: # Error: an error was thrown while running the Action (or Action middleware) action_response = ActionResponse( action=action_request.action, errors=e.errors, ) else: # Error: Action not found. action_response = ActionResponse( action=action_request.action, errors=[Error( code=ERROR_CODE_UNKNOWN, message='The action "{}" was not found on this server.'.format(action_request.action), field='action', )], ) job_response.actions.append(action_response) if ( action_response.errors and not job_request['control'].get('continue_on_error', False) ): # Quit running Actions if an error occurred and continue_on_error is False break return job_response
def wrapped_send_request(client, service_name, actions, *args, **kwargs): assert isinstance( service_name, six.text_type ), 'Called service name "{}" must be unicode'.format( service_name, ) actions_to_send_to_mock = OrderedDict() actions_to_send_to_wrapped_client = [] for i, action_request in enumerate(actions): action_name = getattr(action_request, 'action', None) or action_request['action'] assert isinstance( action_name, six.text_type ), 'Called action name "{}" must be unicode'.format( action_name, ) if service_name == self.service and action_name == self.action: # If the service AND action name match, we should send the request to our mocked client if not isinstance(action_request, ActionRequest): action_request = ActionRequest(**action_request) actions_to_send_to_mock[i] = action_request else: # If the service OR action name DO NOT match, we should delegate the request to the wrapped client actions_to_send_to_wrapped_client.append(action_request) request_id = _global_stub_action_request_counter.get_next() continue_on_error = kwargs.get('continue_on_error', False) if actions_to_send_to_wrapped_client: # If any un-stubbed actions need to be sent to the original client, send them self._services_with_calls_sent_to_wrapped_client.add( service_name) unwrapped_request_id = self._wrapped_client_send_request( client, service_name, actions_to_send_to_wrapped_client, *args, **kwargs) if not actions_to_send_to_mock: # If there are no stubbed actions to mock, just return the un-stubbed request ID return unwrapped_request_id self._stub_action_responses_to_merge[service_name][ unwrapped_request_id] = ( request_id, continue_on_error, ) ordered_actions_for_merging = OrderedDict() job_response_transport_exception = None job_response = JobResponse() for i, action_request in actions_to_send_to_mock.items(): mock_response = None try: mock_response = mock_action(action_request.body or {}) if isinstance(mock_response, JobResponse): job_response.errors.extend(mock_response.errors) if mock_response.actions: mock_response = mock_response.actions[0] elif isinstance(mock_response, dict): mock_response = ActionResponse(self.action, body=mock_response) elif not isinstance(mock_response, ActionResponse): mock_response = ActionResponse(self.action) except ActionError as e: mock_response = ActionResponse(self.action, errors=e.errors) except JobError as e: job_response.errors.extend(e.errors) except (MessageReceiveError, MessageReceiveTimeout) as e: job_response_transport_exception = e if mock_response: ordered_actions_for_merging[i] = mock_response job_response.actions.append(mock_response) if not continue_on_error and mock_response.errors: break if job_response.errors: break if actions_to_send_to_wrapped_client: # If the responses will have to be merged by get_all_responses, replace the list with the ordered dict job_response.actions = ordered_actions_for_merging self._stub_action_responses_outstanding[service_name][ request_id] = (job_response_transport_exception or job_response) return request_id
def wrapped(client, service_name, actions, *args, **kwargs): assert isinstance( service_name, six.text_type ), 'Called service name "{}" must be unicode'.format( service_name, ) requests_to_send_to_mock_client = OrderedDict() requests_to_send_to_wrapped_client = [] for i, action_request in enumerate(actions): action_name = getattr(action_request, 'action', None) or action_request['action'] assert isinstance( action_name, six.text_type ), 'Called action name "{}" must be unicode'.format( action_name, ) if service_name == self.service and action_name == self.action: # If the service AND action name match, we should send the request to our mocked client if not isinstance(action_request, ActionRequest): action_request = ActionRequest(action_request) requests_to_send_to_mock_client[i] = action_request else: # If the service OR action name DO NOT match, we should delegate the request to the wrapped client requests_to_send_to_wrapped_client.append(action_request) # Hold off on raising action errors until both mock and real responses are merged raise_action_errors = kwargs.get('raise_action_errors', True) kwargs['raise_action_errors'] = False # Run the real and mocked jobs and merge the results, to simulate a single job if requests_to_send_to_wrapped_client: job_response = self._wrapped_client_call_actions_method( client, service_name, requests_to_send_to_wrapped_client, *args, **kwargs) else: job_response = JobResponse() for i, action_request in requests_to_send_to_mock_client.items(): try: mock_response = mock_action(action_request.body or {}) if isinstance(mock_response, JobResponse): mock_response = mock_response.actions[0] elif isinstance(mock_response, dict): mock_response = ActionResponse(self.action, body=mock_response) elif not isinstance(mock_response, ActionResponse): mock_response = ActionResponse(self.action) except ActionError as e: mock_response = ActionResponse(self.action, errors=e.errors) job_response.actions.insert(i, mock_response) if kwargs.get('continue_on_error', False) is False: # Simulate the server job halting on the first action error first_error_index = -1 for i, action_result in enumerate(job_response.actions): if action_result.errors: first_error_index = i break if first_error_index >= 0: job_response.actions = job_response.actions[: first_error_index + 1] if raise_action_errors: error_actions = [ action for action in job_response.actions if action.errors ] if error_actions: raise Client.CallActionError(error_actions) return job_response