def ClientServerCommunicate(self, timestamp=None):
"""Tests the end to end encrypted communicators."""
message_list = rdf_flows.MessageList()
for i in range(1, 11):
message_list.job.Append(
session_id=rdfvalue.SessionID(
base="aff4:/flows", queue=queues.FLOWS, flow_name=i),
name="OMG it's a string")
result = rdf_flows.ClientCommunication()
timestamp = self.client_communicator.EncodeMessages(
message_list, result, timestamp=timestamp)
self.cipher_text = result.SerializeToBytes()
(decoded_messages, source, client_timestamp) = (
self.server_communicator.DecryptMessage(self.cipher_text))
self.assertEqual(source, self.client_communicator.common_name)
self.assertEqual(client_timestamp, timestamp)
self.assertLen(decoded_messages, 10)
for i in range(1, 11):
self.assertEqual(
decoded_messages[i - 1].session_id,
rdfvalue.SessionID(
base="aff4:/flows", queue=queues.FLOWS, flow_name=i))
return decoded_messages
def testErrorDetection(self):
"""Tests the end to end encrypted communicators."""
# Install the client - now we can verify its signed messages
self._MakeClientRecord()
# Make something to send
message_list = rdf_flows.MessageList()
for i in range(0, 10):
message_list.job.Append(session_id=str(i))
result = rdf_flows.ClientCommunication()
self.client_communicator.EncodeMessages(message_list, result)
# TODO: We use `bytes` from the `future` package here to have
# Python 3 iteration behaviour. This call should be a noop in Python 3 and
# should be safe to remove on support for Python 2 is dropped.
cipher_text = bytes(result.SerializeToBytes())
# Depending on this modification several things may happen:
# 1) The padding may not match which will cause a decryption exception.
# 2) The protobuf may fail to decode causing a decoding exception.
# 3) The modification may affect the signature resulting in UNAUTHENTICATED
# messages.
# 4) The modification may have no effect on the data at all.
for x in range(0, len(cipher_text), 50):
# Futz with the cipher text (Make sure it's really changed)
mod = chr((cipher_text[x] % 250) + 1).encode("latin-1")
mod_cipher_text = cipher_text[:x] + mod + cipher_text[x + 1:]
# TODO: Now we revert back to native `bytes` object because
# proto deserialization assumes native indexing behaviour.
if compatibility.PY2:
mod_cipher_text = mod_cipher_text.__native__()
try:
decoded, client_id, _ = self.server_communicator.DecryptMessage(
mod_cipher_text)
for i, message in enumerate(decoded):
# If the message is actually authenticated it must not be changed!
if message.auth_state == message.AuthorizationState.AUTHENTICATED:
self.assertEqual(message.source, client_id)
# These fields are set by the decoder and are not present in the
# original message - so we clear them before comparison.
message.auth_state = None
message.source = None
self.assertEqual(message, message_list.job[i])
else:
logging.debug("Message %s: Authstate: %s", i,
message.auth_state)
except communicator.DecodingError as e:
logging.debug("Detected alteration at %s: %s", x, e)
def __init__(self, response_comms, private_key):
self.private_key = private_key
self.response_comms = response_comms
if response_comms.api_version not in [3]:
raise DecryptionError("Unsupported api version: %s, expected 3." %
response_comms.api_version)
if not response_comms.encrypted_cipher:
# The message is not encrypted. We do not allow unencrypted
# messages:
raise DecryptionError("Server response is not encrypted.")
try:
# The encrypted_cipher contains the session key, iv and hmac_key.
self.serialized_cipher = private_key.Decrypt(
response_comms.encrypted_cipher)
# If we get here we have the session keys.
self.cipher = rdf_flows.CipherProperties.FromSerializedBytes(
self.serialized_cipher)
# Check the key lengths.
if (len(self.cipher.key) * 8 != self.key_size
or len(self.cipher.metadata_iv) * 8 != self.iv_size
or len(self.cipher.hmac_key) * 8 != self.key_size):
raise DecryptionError("Invalid cipher.")
self.VerifyHMAC()
# Cipher_metadata contains information about the cipher - It is encrypted
# using the symmetric session key. It contains the RSA signature of the
# digest of the serialized CipherProperties(). It is stored inside the
# encrypted payload.
serialized_metadata = self.Decrypt(
response_comms.encrypted_cipher_metadata,
self.cipher.metadata_iv)
self.cipher_metadata = rdf_flows.CipherMetadata.FromSerializedBytes(
serialized_metadata)
except (rdf_crypto.InvalidSignature, rdf_crypto.CipherError) as e:
if "Ciphertext length must be equal to key size" in str(e):
logging.warning(
e) # Print original stack trace for investigation.
logging.warning("Error for HTTP request %s",
response_comms.orig_request)
# Also log ClientCommunication object, but strip out huge payload.
comms_info = rdf_flows.ClientCommunication(response_comms)
comms_info.encrypted = None
logging.warning("Error for ClientCommunication %s", comms_info)
raise LegacyClientDecryptionError(e)
else:
raise DecryptionError(e)
def UrlMock(self, num_messages=10, url=None, data=None, **kwargs):
"""A mock for url handler processing from the server's POV."""
if "server.pem" in url:
cert = str(config.CONFIG["Frontend.certificate"]).encode("ascii")
return MakeResponse(200, cert)
_ = kwargs
try:
comms_cls = rdf_flows.ClientCommunication
self.client_communication = comms_cls.FromSerializedBytes(data)
# Decrypt incoming messages
self.messages, source, ts = self.server_communicator.DecodeMessages(
self.client_communication)
# Make sure the messages are correct
self.assertEqual(source, self.client_cn)
messages = sorted([
m for m in self.messages if m.session_id == "aff4:/W:session"
],
key=lambda m: m.response_id)
self.assertEqual([m.response_id for m in messages],
list(range(len(messages))))
self.assertEqual([m.request_id for m in messages],
[1] * len(messages))
# Now prepare a response
response_comms = rdf_flows.ClientCommunication()
message_list = rdf_flows.MessageList()
for i in range(0, num_messages):
message_list.job.Append(request_id=i, **self.server_response)
# Preserve the timestamp as a nonce
self.server_communicator.EncodeMessages(
message_list,
response_comms,
destination=source,
timestamp=ts,
api_version=self.client_communication.api_version)
return MakeResponse(200, response_comms.SerializeToBytes())
except communicator.UnknownClientCertError:
raise MakeHTTPException(406)
except Exception as e:
logging.info("Exception in mock urllib.request.Open: %s.", e)
self.last_urlmock_error = e
if flags.FLAGS.pdb_post_mortem:
pdb.post_mortem()
raise MakeHTTPException(500)
def testErrorDetection(self):
"""Tests the end to end encrypted communicators."""
# Install the client - now we can verify its signed messages
self._MakeClientRecord()
# Make something to send
message_list = rdf_flows.MessageList()
for i in range(0, 10):
message_list.job.Append(session_id=str(i))
result = rdf_flows.ClientCommunication()
self.client_communicator.EncodeMessages(message_list, result)
cipher_text = result.SerializeToString()
# Depending on this modification several things may happen:
# 1) The padding may not match which will cause a decryption exception.
# 2) The protobuf may fail to decode causing a decoding exception.
# 3) The modification may affect the signature resulting in UNAUTHENTICATED
# messages.
# 4) The modification may have no effect on the data at all.
for x in range(0, len(cipher_text), 50):
# Futz with the cipher text (Make sure it's really changed)
mod = chr((ord(cipher_text[x]) % 250) + 1).encode("latin-1")
mod_cipher_text = cipher_text[:x] + mod + cipher_text[x + 1:]
try:
decoded, client_id, _ = self.server_communicator.DecryptMessage(
mod_cipher_text)
for i, message in enumerate(decoded):
# If the message is actually authenticated it must not be changed!
if message.auth_state == message.AuthorizationState.AUTHENTICATED:
self.assertEqual(message.source, client_id)
# These fields are set by the decoder and are not present in the
# original message - so we clear them before comparison.
message.auth_state = None
message.source = None
self.assertEqual(message, message_list.job[i])
else:
logging.debug("Message %s: Authstate: %s", i,
message.auth_state)
except communicator.DecodingError as e:
logging.debug("Detected alteration at %s: %s", x, e)
def Control(self):
"""Handle POSTS."""
# Get the api version
try:
api_version = int(
urlparse.parse_qs(self.path.split("?")[1])["api"][0])
except (ValueError, KeyError, IndexError):
# The oldest api version we support if not specified.
api_version = 3
try:
if compatibility.PY2:
content_length = self.headers.getheader("content-length")
else:
content_length = self.headers.get("content-length")
if not content_length:
raise IOError("No content-length header provided.")
length = int(content_length)
request_comms = rdf_flows.ClientCommunication.FromSerializedBytes(
self._GetPOSTData(length))
# If the client did not supply the version in the protobuf we use the get
# parameter.
if not request_comms.api_version:
request_comms.api_version = api_version
# Reply using the same version we were requested with.
responses_comms = rdf_flows.ClientCommunication(
api_version=request_comms.api_version)
# TODO: Python's documentation is just plain terrible and
# does not explain what `client_address` exactly is or what type does it
# have (because its Python, why would they bother) so just to be on the
# safe side, we anticipate byte-string addresses in Python 2 and convert
# that if needed. On Python 3 these should be always unicode strings, so
# once support for Python 2 is dropped this branch can be removed.
address = self.client_address[0]
if compatibility.PY2 and isinstance(self.client_address[0], bytes):
address = address.decode("ascii")
source_ip = ipaddress.ip_address(address)
if source_ip.version == 6:
source_ip = source_ip.ipv4_mapped or source_ip
request_comms.orig_request = rdf_flows.HttpRequest(
timestamp=rdfvalue.RDFDatetime.Now(),
raw_headers=str(self.headers),
source_ip=str(source_ip))
source, nr_messages = self.server.frontend.HandleMessageBundles(
request_comms, responses_comms)
server_logging.LOGGER.LogHttpFrontendAccess(
request_comms.orig_request,
source=source,
message_count=nr_messages)
self.Send(responses_comms.SerializeToBytes())
except communicator.UnknownClientCertError:
# "406 Not Acceptable: The server can only generate a response that is not
# accepted by the client". This is because we can not encrypt for the
# client appropriately.
self.Send(b"Enrollment required", status=406)
def Control(self):
"""Handle POSTS."""
if not master.MASTER_WATCHER.IsMaster():
# We shouldn't be getting requests from the client unless we
# are the active instance.
stats_collector_instance.Get().IncrementCounter(
"frontend_inactive_request_count", fields=["http"])
logging.info("Request sent to inactive frontend from %s",
self.client_address[0])
# Get the api version
try:
api_version = int(cgi.parse_qs(self.path.split("?")[1])["api"][0])
except (ValueError, KeyError, IndexError):
# The oldest api version we support if not specified.
api_version = 3
try:
content_length = self.headers.getheader("content-length")
if not content_length:
raise IOError("No content-length header provided.")
length = int(content_length)
request_comms = rdf_flows.ClientCommunication.FromSerializedString(
self._GetPOSTData(length))
# If the client did not supply the version in the protobuf we use the get
# parameter.
if not request_comms.api_version:
request_comms.api_version = api_version
# Reply using the same version we were requested with.
responses_comms = rdf_flows.ClientCommunication(
api_version=request_comms.api_version)
source_ip = ipaddr.IPAddress(self.client_address[0])
if source_ip.version == 6:
source_ip = source_ip.ipv4_mapped or source_ip
request_comms.orig_request = rdf_flows.HttpRequest(
timestamp=rdfvalue.RDFDatetime.Now().AsMicrosecondsSinceEpoch(
),
raw_headers=utils.SmartStr(self.headers),
source_ip=utils.SmartStr(source_ip))
source, nr_messages = self.server.frontend.HandleMessageBundles(
request_comms, responses_comms)
server_logging.LOGGER.LogHttpFrontendAccess(
request_comms.orig_request,
source=source,
message_count=nr_messages)
self.Send(responses_comms.SerializeToString())
except communicator.UnknownClientCert:
# "406 Not Acceptable: The server can only generate a response that is not
# accepted by the client". This is because we can not encrypt for the
# client appropriately.
self.Send("Enrollment required", status=406)
def RunOnce(self):
"""Makes a single request to the GRR server.
Returns:
A Status() object indicating how the last POST went.
"""
# Attempt to fetch and load server certificate.
if not self._FetchServerCertificate():
self.timer.Wait()
return HTTPObject(code=500)
# Here we only drain messages if we were able to connect to the server in
# the last poll request. Otherwise we just wait until the connection comes
# back so we don't expire our messages too fast.
if self.http_manager.consecutive_connection_errors == 0:
# Grab some messages to send
message_list = self.client_worker.Drain(
max_size=config.CONFIG["Client.max_post_size"])
else:
message_list = rdf_flows.MessageList()
# If any outbound messages require fast poll we switch to fast poll mode.
for message in message_list.job:
if message.require_fastpoll:
self.timer.FastPoll()
break
# Make new encrypted ClientCommunication rdfvalue.
payload = rdf_flows.ClientCommunication()
# If our memory footprint is too large, we advertise that our input queue
# is full. This will prevent the server from sending us any messages, and
# hopefully allow us to work down our memory usage, by processing any
# outstanding messages.
if self.client_worker.MemoryExceeded():
logging.info("Memory exceeded, will not retrieve jobs.")
payload.queue_size = 1000000
else:
# Let the server know how many messages are currently queued in
# the input queue.
payload.queue_size = self.client_worker.InQueueSize()
nonce = self.communicator.EncodeMessages(message_list, payload)
payload_data = payload.SerializeToString()
response = self.MakeRequest(payload_data)
# Unable to decode response or response not valid.
if response.code != 200 or response.messages is None:
# We don't print response here since it should be encrypted and will
# cause ascii conversion errors.
logging.info("%s: Could not connect to server at %s, status %s",
self.communicator.common_name,
self.http_manager.active_base_url, response.code)
# Force the server pem to be reparsed on the next connection.
self.server_certificate = None
# Reschedule the tasks back on the queue so they get retried next time.
messages = list(message_list.job)
for message in messages:
message.require_fastpoll = False
message.ttl -= 1
if message.ttl > 0:
self.client_worker.QueueResponse(message)
else:
logging.info("Dropped message due to retransmissions.")
return response
# Check the decoded nonce was as expected.
if response.nonce != nonce:
logging.info("Nonce not matched.")
response.code = 500
return response
if response.source != self.communicator.server_name:
logging.info("Received a message not from the server "
"%s, expected %s.", response.source,
self.communicator.server_name)
response.code = 500
return response
# Check to see if any inbound messages want us to fastpoll. This means we
# drop to fastpoll immediately on a new request rather than waiting for the
# next beacon to report results.
for message in response.messages:
if message.require_fastpoll:
self.timer.FastPoll()
break
# Process all messages. Messages can be processed by clients in
# any order since clients do not have state.
self.client_worker.QueueMessages(response.messages)
cn = self.communicator.common_name
logging.info(
"%s: Sending %s(%s), Received %s messages in %s sec. "
"Sleeping for %s sec.", cn, len(message_list), len(payload_data),
len(response.messages), response.duration, self.timer.sleep_time)
return response