def handle(self, environ, start_response): """The request handler.""" if not master.MASTER_WATCHER.IsMaster(): # We shouldn't be getting requests from the client unless we # are the active instance. stats.STATS.IncrementCounter("frontend_inactive_request_count", fields=["http"]) logging.info("Request sent to inactive frontend") if environ["REQUEST_METHOD"] == "GET": if environ["PATH_INFO"] == "/server.pem": return self.Send(self.server_pem, start_response) else: return self.Send("", start_response) if environ["REQUEST_METHOD"] == "POST": try: length = int(environ["CONTENT_LENGTH"]) input_data = environ["wsgi.input"].read(length) request_comms = rdf_flows.ClientCommunication(input_data) responses_comms = rdf_flows.ClientCommunication() self.front_end.HandleMessageBundles(request_comms, responses_comms) return self.Send(responses_comms.SerializeToString(), start_response) except communicator.UnknownClientCert: return self.Send("Enrollment required", start_response, "406 Not acceptable")
def UrlMock(self, req, num_messages=10, **kwargs): """A mock for url handler processing from the server's POV.""" if "server.pem" in req.get_full_url(): return StringIO.StringIO(config_lib.CONFIG["Frontend.certificate"]) _ = kwargs try: self.client_communication = rdf_flows.ClientCommunication(req.data) # Decrypt incoming messages self.messages, source, ts = self.server_communicator.DecodeMessages( self.client_communication) # Make sure the messages are correct self.assertEqual(source, self.client_cn) for i, message in enumerate(self.messages): # Do not check any status messages. if message.request_id: self.assertEqual(message.response_id, i) self.assertEqual(message.request_id, 1) self.assertEqual(message.session_id, "aff4:/W:session") # Now prepare a response response_comms = rdf_flows.ClientCommunication() message_list = rdf_flows.MessageList() for i in range(0, num_messages): message_list.job.Append(request_id=i, **self.server_response) # Preserve the timestamp as a nonce self.server_communicator.EncodeMessages( message_list, response_comms, destination=source, timestamp=ts, api_version=self.client_communication.api_version) return StringIO.StringIO(response_comms.SerializeToString()) except communicator.UnknownClientCert: raise urllib2.HTTPError(url=None, code=406, msg=None, hdrs=None, fp=None) except Exception as e: logging.info("Exception in mock urllib2.Open: %s.", e) self.last_urlmock_error = e if flags.FLAGS.debug: pdb.post_mortem() raise urllib2.HTTPError(url=None, code=500, msg=None, hdrs=None, fp=None)
def ClientServerCommunicate(self, timestamp=None): """Tests the end to end encrypted communicators.""" message_list = rdf_flows.MessageList() for i in range(1, 11): message_list.job.Append( session_id=rdfvalue.SessionID( base="aff4:/flows", queue=queues.FLOWS, flow_name=i), name="OMG it's a string") result = rdf_flows.ClientCommunication() timestamp = self.client_communicator.EncodeMessages( message_list, result, timestamp=timestamp) self.cipher_text = result.SerializeToString() (decoded_messages, source, client_timestamp) = ( self.server_communicator.DecryptMessage(self.cipher_text)) self.assertEqual(source, self.client_communicator.common_name) self.assertEqual(client_timestamp, timestamp) self.assertEqual(len(decoded_messages), 10) for i in range(1, 11): self.assertEqual( decoded_messages[i - 1].session_id, rdfvalue.SessionID( base="aff4:/flows", queue=queues.FLOWS, flow_name=i)) return decoded_messages
def Corruptor(req, **_): """Futz with some of the fields.""" self.client_communication = rdf_flows.ClientCommunication(req.data) if self.corruptor_field and "server.pem" not in req.get_full_url(): orig_str_repr = self.client_communication.SerializeToString() field_data = getattr(self.client_communication, self.corruptor_field) if hasattr(field_data, "SerializeToString"): # This converts encryption keys to a string so we can corrupt them. field_data = field_data.SerializeToString() modified_data = array.array("c", field_data) offset = len(field_data) / 2 modified_data[offset] = chr((ord(field_data[offset]) % 250) + 1) setattr(self.client_communication, self.corruptor_field, modified_data.tostring()) # Make sure we actually changed the data. self.assertNotEqual(field_data, modified_data) mod_str_repr = self.client_communication.SerializeToString() self.assertEqual(len(orig_str_repr), len(mod_str_repr)) differences = [ True for x, y in zip(orig_str_repr, mod_str_repr) if x != y ] self.assertEqual(len(differences), 1) req.data = self.client_communication.SerializeToString() return self.UrlMock(req)
def testHandleMessageBundle(self): """Check that HandleMessageBundles() requeues messages if it failed. This test makes sure that when messages are pending for a client, and which we have no certificate for, the messages are requeued when sending fails. """ # Make a new fake client client_id = self.SetupClient(0) class MockCommunicator(object): """A fake that simulates an unenrolled client.""" def DecodeMessages(self, *unused_args): """For simplicity client sends an empty request.""" return ([], client_id, 100) def EncodeMessages(self, *unused_args, **unused_kw): """Raise because the server has no certificates for this client.""" raise communicator.UnknownClientCert() # Install the mock. self.server._communicator = MockCommunicator() # First request, the server will raise UnknownClientCert. request_comms = rdf_flows.ClientCommunication() self.assertRaises(communicator.UnknownClientCert, self.server.HandleMessageBundles, request_comms, 2) # We can still schedule a flow for it flow.GRRFlow.StartFlow( client_id=client_id, flow_name=flow_test_lib.SendingFlow.__name__, message_count=1, token=self.token) manager = queue_manager.QueueManager(token=self.token) tasks = manager.Query(client_id, limit=100) self.assertRaises(communicator.UnknownClientCert, self.server.HandleMessageBundles, request_comms, 2) new_tasks = manager.Query(client_id, limit=100) # The different in eta times reflect the lease that the server took on the # client messages. lease_time = (new_tasks[0].eta - tasks[0].eta) / 1e6 # This lease time must be small, as the HandleMessageBundles() call failed, # the pending client messages must be put back on the queue. self.assertLess(lease_time, 1) # Since the server tried to send it, the ttl must be decremented self.assertEqual(tasks[0].task_ttl - new_tasks[0].task_ttl, 1)
def DecryptMessage(self, encrypted_response): """Decrypt the serialized, encrypted string. Args: encrypted_response: A serialized and encrypted string. Returns: a Signed_Message_List rdfvalue """ try: response_comms = rdf_flows.ClientCommunication(encrypted_response) return self.DecodeMessages(response_comms) except (rdfvalue.DecodeError, type_info.TypeValueError, ValueError, AttributeError) as e: raise DecodingError("Protobuf parsing error: %s" % e)
def testErrorDetection(self): """Tests the end to end encrypted communicators.""" # Install the client - now we can verify its signed messages self.MakeClientAFF4Record() # Make something to send message_list = rdf_flows.MessageList() for i in range(0, 10): message_list.job.Append(session_id=str(i)) result = rdf_flows.ClientCommunication() self.client_communicator.EncodeMessages(message_list, result) cipher_text = result.SerializeToString() # Depending on this modification several things may happen: # 1) The padding may not match which will cause a decryption exception. # 2) The protobuf may fail to decode causing a decoding exception. # 3) The modification may affect the signature resulting in UNAUTHENTICATED # messages. # 4) The modification may have no effect on the data at all. for x in range(0, len(cipher_text), 50): # Futz with the cipher text (Make sure it's really changed) mod_cipher_text = (cipher_text[:x] + chr((ord(cipher_text[x]) % 250) + 1) + cipher_text[x + 1:]) try: decoded, client_id, _ = self.server_communicator.DecryptMessage( mod_cipher_text) for i, message in enumerate(decoded): # If the message is actually authenticated it must not be changed! if message.auth_state == message.AuthorizationState.AUTHENTICATED: self.assertEqual(message.source, client_id) # These fields are set by the decoder and are not present in the # original message - so we clear them before comparison. message.auth_state = None message.source = None self.assertRDFValuesEqual(message, message_list.job[i]) else: logging.debug("Message %s: Authstate: %s", i, message.auth_state) except communicator.DecodingError as e: logging.debug("Detected alteration at %s: %s", x, e)
def Corruptor(req, **_): """Futz with some of the fields.""" self.client_communication = rdf_flows.ClientCommunication(req.data) if self.corruptor_field and "server.pem" not in req.get_full_url(): field_data = getattr(self.client_communication, self.corruptor_field) modified_data = array.array("c", field_data) offset = len(field_data) / 2 modified_data[offset] = chr((ord(field_data[offset]) % 250) + 1) setattr(self.client_communication, self.corruptor_field, str(modified_data)) # Make sure we actually changed the data. self.assertNotEqual(field_data, modified_data) req.data = self.client_communication.SerializeToString() return self.UrlMock(req)
def Control(self): """Handle POSTS.""" if not master.MASTER_WATCHER.IsMaster(): # We shouldn't be getting requests from the client unless we # are the active instance. stats.STATS.IncrementCounter("frontend_inactive_request_count", fields=["http"]) logging.info("Request sent to inactive frontend from %s", self.client_address[0]) # Get the api version try: api_version = int(cgi.parse_qs(self.path.split("?")[1])["api"][0]) except (ValueError, KeyError, IndexError): # The oldest api version we support if not specified. api_version = 3 with GRRHTTPServerHandler.active_counter_lock: GRRHTTPServerHandler.active_counter += 1 stats.STATS.SetGaugeValue("frontend_active_count", self.active_counter, fields=["http"]) try: length = int(self.headers.getheader("content-length")) request_comms = rdf_flows.ClientCommunication(self._GetPOSTData(length)) # If the client did not supply the version in the protobuf we use the get # parameter. if not request_comms.api_version: request_comms.api_version = api_version # Reply using the same version we were requested with. responses_comms = rdf_flows.ClientCommunication( api_version=request_comms.api_version) source_ip = ipaddr.IPAddress(self.client_address[0]) if source_ip.version == 6: source_ip = source_ip.ipv4_mapped or source_ip request_comms.orig_request = rdf_flows.HttpRequest( raw_headers=utils.SmartStr(self.headers), source_ip=utils.SmartStr(source_ip)) source, nr_messages = self.server.frontend.HandleMessageBundles( request_comms, responses_comms) logging.info("HTTP request from %s (%s), %d bytes - %d messages received," " %d messages sent.", source, utils.SmartStr(source_ip), length, nr_messages, responses_comms.num_messages) self.Send(responses_comms.SerializeToString()) except communicator.UnknownClientCert: # "406 Not Acceptable: The server can only generate a response that is not # accepted by the client". This is because we can not encrypt for the # client appropriately. self.Send("Enrollment required", status=406) except Exception as e: # pylint: disable=broad-except if flags.FLAGS.debug: pdb.post_mortem() logging.error("Had to respond with status 500: %s.", e) self.Send("Error", status=500) finally: with GRRHTTPServerHandler.active_counter_lock: GRRHTTPServerHandler.active_counter -= 1 stats.STATS.SetGaugeValue("frontend_active_count", self.active_counter, fields=["http"])
def Control(self): """Handle POSTS.""" if not master.MASTER_WATCHER.IsMaster(): # We shouldn't be getting requests from the client unless we # are the active instance. stats.STATS.IncrementCounter("frontend_inactive_request_count", fields=["http"]) logging.info("Request sent to inactive frontend from %s", self.client_address[0]) # Get the api version try: api_version = int(cgi.parse_qs(self.path.split("?")[1])["api"][0]) except (ValueError, KeyError, IndexError): # The oldest api version we support if not specified. api_version = 3 with GRRHTTPServerHandler.active_counter_lock: GRRHTTPServerHandler.active_counter += 1 stats.STATS.SetGaugeValue("frontend_active_count", self.active_counter, fields=["http"]) try: content_length = self.headers.getheader("content-length") if not content_length: raise IOError("No content-length header provided.") length = int(content_length) request_comms = rdf_flows.ClientCommunication.FromSerializedString( self._GetPOSTData(length)) # If the client did not supply the version in the protobuf we use the get # parameter. if not request_comms.api_version: request_comms.api_version = api_version # Reply using the same version we were requested with. responses_comms = rdf_flows.ClientCommunication( api_version=request_comms.api_version) source_ip = ipaddr.IPAddress(self.client_address[0]) if source_ip.version == 6: source_ip = source_ip.ipv4_mapped or source_ip request_comms.orig_request = rdf_flows.HttpRequest( timestamp=rdfvalue.RDFDatetime.Now().AsMicrosecondsSinceEpoch( ), raw_headers=utils.SmartStr(self.headers), source_ip=utils.SmartStr(source_ip)) source, nr_messages = self.server.frontend.HandleMessageBundles( request_comms, responses_comms) server_logging.LOGGER.LogHttpFrontendAccess( request_comms.orig_request, source=source, message_count=nr_messages) self.Send(responses_comms.SerializeToString()) except communicator.UnknownClientCert: # "406 Not Acceptable: The server can only generate a response that is not # accepted by the client". This is because we can not encrypt for the # client appropriately. self.Send("Enrollment required", status=406) finally: with GRRHTTPServerHandler.active_counter_lock: GRRHTTPServerHandler.active_counter -= 1 stats.STATS.SetGaugeValue("frontend_active_count", self.active_counter, fields=["http"])
def RunOnce(self): """Makes a single request to the GRR server. Returns: A Status() object indicating how the last POST went. """ try: status = Status() # Here we only drain messages if we were able to connect to the server in # the last poll request. Otherwise we just wait until the connection comes # back so we don't expire our messages too fast. if self.consecutive_connection_errors == 0: # Grab some messages to send message_list = self.client_worker.Drain( max_size=config_lib.CONFIG["Client.max_post_size"]) else: message_list = rdf_flows.MessageList() sent_count = 0 sent = {} require_fastpoll = False for message in message_list.job: sent_count += 1 require_fastpoll |= message.require_fastpoll sent.setdefault(message.priority, 0) sent[message.priority] += 1 status = Status(sent_count=sent_count, sent=sent, require_fastpoll=require_fastpoll) # Make new encrypted ClientCommunication rdfvalue. payload = rdf_flows.ClientCommunication() # If our memory footprint is too large, we advertise that our input queue # is full. This will prevent the server from sending us any messages, and # hopefully allow us to work down our memory usage, by processing any # outstanding messages. if self.client_worker.MemoryExceeded(): logging.info("Memory exceeded, will not retrieve jobs.") payload.queue_size = 1000000 else: # Let the server know how many messages are currently queued in # the input queue. payload.queue_size = self.client_worker.InQueueSize() nonce = self.communicator.EncodeMessages(message_list, payload) response = self.MakeRequest(payload.SerializeToString(), status) if status.code != 200: # We don't print response here since it should be encrypted and will # cause ascii conversion errors. logging.info( "%s: Could not connect to server at %s, status %s", self.communicator.common_name, self.GetServerUrl(), status.code) # Reschedule the tasks back on the queue so they get retried next time. messages = list(message_list.job) for message in messages: message.priority = rdf_flows.GrrMessage.Priority.HIGH_PRIORITY message.require_fastpoll = False message.ttl -= 1 if message.ttl > 0: # Schedule with high priority to make it jump the queue. self.client_worker.QueueResponse( message, rdf_flows.GrrMessage.Priority.HIGH_PRIORITY + 1) else: logging.info("Dropped message due to retransmissions.") return status if not response: return status try: tmp = self.communicator.DecryptMessage(response) (messages, source, server_nonce) = tmp if server_nonce != nonce: logging.info("Nonce not matched.") status.code = 500 return status except proto2_message.DecodeError: logging.info("Protobuf decode error. Bad URL or auth.") status.code = 500 return status if source != self.communicator.server_name: logging.info( "Received a message not from the server " "%s, expected %s.", source, self.communicator.server_name) status.code = 500 return status status.received_count = len(messages) # If we're not going to fastpoll based on outbound messages, check to see # if any inbound messages want us to fastpoll. This means we drop to # fastpoll immediately on a new request rather than waiting for the next # beacon to report results. if not status.require_fastpoll: for message in messages: if message.require_fastpoll: status.require_fastpoll = True break # Process all messages. Messages can be processed by clients in # any order since clients do not have state. self.client_worker.QueueMessages(messages) except Exception: # pylint: disable=broad-except # Catch everything, yes, this is terrible but necessary logging.warn("Uncaught exception caught: %s", traceback.format_exc()) if status: status.code = 500 if flags.FLAGS.debug: pdb.post_mortem() return status