async def _decrypt_media(
self,
data: AsyncIterator[bytes],
info: EncryptedMediaInfo,
) -> AsyncIterator[bytes]:
try:
wanted_sha256 = decode_base64(info.sha256)
aes256_key = decode_base64(info.key)
init_vector = decode_base64(info.init_vector)
except BinAsciiError as e:
raise err.MediaInvalidBase64(next(iter(e.args), ""))
sha256 = SHA256.new()
prefix = init_vector[:8]
init_value = int.from_bytes(init_vector[8:], "big")
counter = Counter.new(64, prefix=prefix, initial_value=init_value)
try:
cipher = AES.new(aes256_key, AES.MODE_CTR, counter=counter)
except ValueError as e:
raise err.MediaAESError(next(iter(e.args), ""))
async for chunk in data:
sha256.update(chunk)
yield cipher.decrypt(chunk)
got_sha256 = sha256.digest()
if wanted_sha256 != got_sha256:
raise err.MediaSHA256Mismatch(wanted_sha256, got_sha256)
def decrypt_attachment(ciphertext, info):
"""Decrypt an encrypted attachment.
Args:
ciphertext (bytes): The data to decrypt.
info (dict): The information needed to decrypt the attachment.
| key: AES-CTR JWK key object.
| iv: Base64 encoded 16 byte AES-CTR IV.
| hashes.sha256: Base64 encoded SHA-256 hash of the ciphertext.
Returns:
The plaintext bytes.
Raises:
RuntimeError if the integrity check fails.
"""
expected_hash = unpaddedbase64.decode_base64(info['hashes']['sha256'])
h = SHA256.new()
h.update(ciphertext)
if h.digest() != expected_hash:
raise RuntimeError('Mismatched SHA-256 digest.')
key = unpaddedbase64.decode_base64(info['key']['k'])
# Drop last 8 bytes, which are 0
iv = unpaddedbase64.decode_base64(info['iv'])[:8]
ctr = Counter.new(64, prefix=iv, initial_value=0)
cipher = AES.new(key, AES.MODE_CTR, counter=ctr)
return cipher.decrypt(ciphertext)
def from_token(cls, token):
if PY3:
# The argument raw=False is only available on new versions of
# msgpack, and only really needed on Python 3. Gate it behind
# a PY3 check to avoid causing issues on Debian-packaged versions.
decoded = msgpack.loads(decode_base64(token), raw=False)
else:
decoded = msgpack.loads(decode_base64(token))
return RoomListNextBatch(
**{cls.REVERSE_KEY_DICT[key]: val
for key, val in decoded.items()})
def from_token(cls, token):
if PY3:
# The argument raw=False is only available on new versions of
# msgpack, and only really needed on Python 3. Gate it behind
# a PY3 check to avoid causing issues on Debian-packaged versions.
decoded = msgpack.loads(decode_base64(token), raw=False)
else:
decoded = msgpack.loads(decode_base64(token))
return RoomListNextBatch(**{
cls.REVERSE_KEY_DICT[key]: val
for key, val in decoded.items()
})
def check_event_content_hash(event, hash_algorithm=hashlib.sha256):
"""Check whether the hash for this PDU matches the contents"""
name, expected_hash = compute_content_hash(event.get_pdu_json(), hash_algorithm)
logger.debug("Expecting hash: %s", encode_base64(expected_hash))
# some malformed events lack a 'hashes'. Protect against it being missing
# or a weird type by basically treating it the same as an unhashed event.
hashes = event.get("hashes")
if not isinstance(hashes, dict):
raise SynapseError(400, "Malformed 'hashes'", Codes.UNAUTHORIZED)
if name not in hashes:
raise SynapseError(
400,
"Algorithm %s not in hashes %s" % (
name, list(hashes),
),
Codes.UNAUTHORIZED,
)
message_hash_base64 = hashes[name]
try:
message_hash_bytes = decode_base64(message_hash_base64)
except Exception:
raise SynapseError(
400,
"Invalid base64: %s" % (message_hash_base64,),
Codes.UNAUTHORIZED,
)
return message_hash_bytes == expected_hash
def check_event_content_hash(event: EventBase,
hash_algorithm: Hasher = hashlib.sha256) -> bool:
"""Check whether the hash for this PDU matches the contents"""
name, expected_hash = compute_content_hash(event.get_pdu_json(),
hash_algorithm)
logger.debug(
"Verifying content hash on %s (expecting: %s)",
event.event_id,
encode_base64(expected_hash),
)
# some malformed events lack a 'hashes'. Protect against it being missing
# or a weird type by basically treating it the same as an unhashed event.
hashes = event.get("hashes")
# nb it might be a frozendict or a dict
if not isinstance(hashes, collections.abc.Mapping):
raise SynapseError(400, "Malformed 'hashes': %s" % (type(hashes), ),
Codes.UNAUTHORIZED)
if name not in hashes:
raise SynapseError(
400,
"Algorithm %s not in hashes %s" % (name, list(hashes)),
Codes.UNAUTHORIZED,
)
message_hash_base64 = hashes[name]
try:
message_hash_bytes = decode_base64(message_hash_base64)
except Exception:
raise SynapseError(400, "Invalid base64: %s" % (message_hash_base64, ),
Codes.UNAUTHORIZED)
return message_hash_bytes == expected_hash
def _parse_key_servers(key_servers, federation_verify_certificates):
try:
jsonschema.validate(key_servers, TRUSTED_KEY_SERVERS_SCHEMA)
except jsonschema.ValidationError as e:
raise ConfigError("Unable to parse 'trusted_key_servers': " +
e.message)
for server in key_servers:
server_name = server["server_name"]
result = TrustedKeyServer(server_name=server_name)
verify_keys = server.get("verify_keys")
if verify_keys is not None:
result.verify_keys = {}
for key_id, key_base64 in verify_keys.items():
if not is_signing_algorithm_supported(key_id):
raise ConfigError(
"Unsupported signing algorithm on key %s for server %s in "
"trusted_key_servers" % (key_id, server_name))
try:
key_bytes = decode_base64(key_base64)
verify_key = decode_verify_key_bytes(key_id, key_bytes)
except Exception as e:
raise ConfigError(
"Unable to parse key %s for server %s in "
"trusted_key_servers: %s" % (key_id, server_name, e))
result.verify_keys[key_id] = verify_key
if not federation_verify_certificates and not server.get(
"accept_keys_insecurely"):
_assert_keyserver_has_verify_keys(result)
yield result
def decrypt(encrypted_payload: str, passphrase: str):
decoded_payload = decode_base64(encrypted_payload)
version = decoded_payload[0]
if isinstance(version, str):
version = ord(version)
if version != 1:
raise ValueError("Unsupported export format version.")
salt = decoded_payload[1:17]
iv = int.from_bytes(decoded_payload[17:33], byteorder="big")
count = int.from_bytes(decoded_payload[33:37], byteorder="big")
encrypted_data = decoded_payload[37:-32]
expected_hmac = decoded_payload[-32:]
derived_key = PBKDF2(passphrase, salt, 64, count, prf) # type: ignore
aes_key = derived_key[:32]
hmac_key = derived_key[32:64]
hmac = HMAC.new(hmac_key, decoded_payload[:-32], SHA256).digest()
if hmac != expected_hmac:
raise ValueError("HMAC check failed for encrypted payload.")
ctr = Counter.new(128, initial_value=iv)
cipher = AES.new(aes_key, AES.MODE_CTR, counter=ctr)
return cipher.decrypt(encrypted_data)
def _verify_third_party_invite(self, event, auth_events):
"""
Validates that the invite event is authorized by a previous third-party invite.
Checks that the public key, and keyserver, match those in the third party invite,
and that the invite event has a signature issued using that public key.
Args:
event: The m.room.member join event being validated.
auth_events: All relevant previous context events which may be used
for authorization decisions.
Return:
True if the event fulfills the expectations of a previous third party
invite event.
"""
if "third_party_invite" not in event.content:
return False
if "signed" not in event.content["third_party_invite"]:
return False
signed = event.content["third_party_invite"]["signed"]
for key in {"mxid", "token"}:
if key not in signed:
return False
token = signed["token"]
invite_event = auth_events.get(
(EventTypes.ThirdPartyInvite, token,)
)
if not invite_event:
return False
if event.user_id != invite_event.user_id:
return False
try:
public_key = invite_event.content["public_key"]
if signed["mxid"] != event.state_key:
return False
if signed["token"] != token:
return False
for server, signature_block in signed["signatures"].items():
for key_name, encoded_signature in signature_block.items():
if not key_name.startswith("ed25519:"):
return False
verify_key = decode_verify_key_bytes(
key_name,
decode_base64(public_key)
)
verify_signed_json(signed, server, verify_key)
# We got the public key from the invite, so we know that the
# correct server signed the signed bundle.
# The caller is responsible for checking that the signing
# server has not revoked that public key.
return True
return False
except (KeyError, SignatureVerifyException,):
return False
def _verify_third_party_invite(self, event, auth_events):
"""
Validates that the invite event is authorized by a previous third-party invite.
Checks that the public key, and keyserver, match those in the third party invite,
and that the invite event has a signature issued using that public key.
Args:
event: The m.room.member join event being validated.
auth_events: All relevant previous context events which may be used
for authorization decisions.
Return:
True if the event fulfills the expectations of a previous third party
invite event.
"""
if "third_party_invite" not in event.content:
return False
if "signed" not in event.content["third_party_invite"]:
return False
signed = event.content["third_party_invite"]["signed"]
for key in {"mxid", "token"}:
if key not in signed:
return False
token = signed["token"]
invite_event = auth_events.get(
(EventTypes.ThirdPartyInvite, token,)
)
if not invite_event:
return False
if event.user_id != invite_event.user_id:
return False
try:
public_key = invite_event.content["public_key"]
if signed["mxid"] != event.state_key:
return False
if signed["token"] != token:
return False
for server, signature_block in signed["signatures"].items():
for key_name, encoded_signature in signature_block.items():
if not key_name.startswith("ed25519:"):
return False
verify_key = decode_verify_key_bytes(
key_name,
decode_base64(public_key)
)
verify_signed_json(signed, server, verify_key)
# We got the public key from the invite, so we know that the
# correct server signed the signed bundle.
# The caller is responsible for checking that the signing
# server has not revoked that public key.
return True
return False
except (KeyError, SignatureVerifyException,):
return False
def decrypt_attachment(ciphertext: bytes, key: str, hash: str,
iv: str) -> bytes:
"""Decrypt an encrypted attachment.
Args:
ciphertext: The data to decrypt.
key: AES_CTR JWK key object.
hash: Base64 encoded SHA-256 hash of the ciphertext.
iv: Base64 encoded 16 byte AES-CTR IV.
Returns:
The plaintext bytes.
Raises:
EncryptionError: if the integrity check fails.
"""
expected_hash = unpaddedbase64.decode_base64(hash)
h = SHA256.new()
h.update(ciphertext)
if h.digest() != expected_hash:
raise DecryptionError("Mismatched SHA-256 digest.")
try:
byte_key: bytes = unpaddedbase64.decode_base64(key)
except (binascii.Error, TypeError):
raise DecryptionError("Error decoding key.")
try:
byte_iv: bytes = unpaddedbase64.decode_base64(iv)
prefix = byte_iv[:8]
# A non-zero IV counter is not spec-compliant, but some clients still do it,
# so decode the counter part too.
initial_value = struct.unpack(">Q", byte_iv[8:])[0]
except (binascii.Error, TypeError, IndexError, struct.error):
raise DecryptionError("Error decoding initial values.")
ctr = Counter.new(64, prefix=prefix, initial_value=initial_value)
try:
cipher = AES.new(byte_key, AES.MODE_CTR, counter=ctr)
except ValueError as e:
raise DecryptionError("Failed to create AES cipher") from e
return cipher.decrypt(ciphertext)
def get_server_keys(server_name, target, port):
url = "https://%s:%i/_matrix/key/v1" % (target, port)
keys = json.load(urllib2.urlopen(url))
verify_keys = {}
for key_id, key_base64 in keys["verify_keys"].items():
verify_key = decode_verify_key_bytes(key_id, decode_base64(key_base64))
verify_signed_json(keys, server_name, verify_key)
verify_keys[key_id] = verify_key
return verify_keys
def get_server_keys(server_name, target, port):
url = "https://%s:%i/_matrix/key/v1" % (target, port)
keys = json.load(urllib2.urlopen(url))
verify_keys = {}
for key_id, key_base64 in keys["verify_keys"].items():
verify_key = decode_verify_key_bytes(key_id, decode_base64(key_base64))
verify_signed_json(keys, server_name, verify_key)
verify_keys[key_id] = verify_key
return verify_keys
def decrypt_attachment(ciphertext: bytes, key: str, hash: str, iv: str):
"""Decrypt an encrypted attachment.
Args:
ciphertext (bytes): The data to decrypt.
key (str): AES_CTR JWK key object.
hash (str): Base64 encoded SHA-256 hash of the ciphertext.
iv (str): Base64 encoded 16 byte AES-CTR IV.
Returns:
The plaintext bytes.
Raises:
EncryptionError if the integrity check fails.
"""
expected_hash = unpaddedbase64.decode_base64(hash)
h = SHA256.new()
h.update(ciphertext)
if h.digest() != expected_hash:
raise EncryptionError("Mismatched SHA-256 digest.")
try:
byte_key: bytes = unpaddedbase64.decode_base64(key)
except (BinAsciiError, TypeError):
raise EncryptionError("Error decoding key.")
try:
byte_iv: bytes = unpaddedbase64.decode_base64(iv)
except (BinAsciiError, TypeError):
raise EncryptionError("Error decoding initial values.")
prefix: bytes = byte_iv[:8]
cnt: int = int.from_bytes(byte_iv[8:], 'big')
ctr = Counter.new(64, prefix=prefix, initial_value=cnt)
try:
cipher = AES.new(byte_key, AES.MODE_CTR, counter=ctr)
except ValueError as e:
raise EncryptionError(e)
return cipher.decrypt(ciphertext)
def decrypt_attachment(ciphertext: bytes, key: str, hash: str,
iv: str) -> bytes:
"""Decrypt an encrypted attachment.
Args:
ciphertext: The data to decrypt.
key: AES_CTR JWK key object.
hash: Base64 encoded SHA-256 hash of the ciphertext.
iv: Base64 encoded 16 byte AES-CTR IV.
Returns:
The plaintext bytes.
Raises:
EncryptionError if the integrity check fails.
"""
expected_hash = unpaddedbase64.decode_base64(hash)
h = SHA256.new()
h.update(ciphertext)
if h.digest() != expected_hash:
raise DecryptionError("Mismatched SHA-256 digest.")
try:
byte_key: bytes = unpaddedbase64.decode_base64(key)
except (BinAsciiError, TypeError):
raise DecryptionError("Error decoding key.")
try:
# Drop last 8 bytes, which are 0
byte_iv: bytes = unpaddedbase64.decode_base64(iv)[:8]
except (BinAsciiError, TypeError):
raise DecryptionError("Error decoding initial values.")
ctr = Counter.new(64, prefix=byte_iv, initial_value=0)
try:
cipher = AES.new(byte_key, AES.MODE_CTR, counter=ctr)
except ValueError as e:
raise DecryptionError("Failed to create AES cipher") from e
return cipher.decrypt(ciphertext)
def read_perspectives(self, perspectives_servers):
servers = {}
for server_name, server_config in perspectives_servers.items():
for key_id, key_data in server_config["verify_keys"].items():
if is_signing_algorithm_supported(key_id):
key_base64 = key_data["key"]
key_bytes = decode_base64(key_base64)
verify_key = decode_verify_key_bytes(key_id, key_bytes)
servers.setdefault(server_name, {})[key_id] = verify_key
return servers
def decrypt_attachment(ciphertext, key, hash, iv):
"""Decrypt an encrypted attachment.
Args:
ciphertext (bytes): The data to decrypt.
key (str): AES_CTR JWK key object.
hash (str): Base64 encoded SHA-256 hash of the ciphertext.
iv (str): Base64 encoded 16 byte AES-CTR IV.
Returns:
The plaintext bytes.
Raises:
EncryptionError if the integrity check fails.
"""
expected_hash = unpaddedbase64.decode_base64(hash)
h = SHA256.new()
h.update(ciphertext)
if h.digest() != expected_hash:
raise EncryptionError("Mismatched SHA-256 digest.")
try:
key = unpaddedbase64.decode_base64(key)
except (base64.binascii.Error, TypeError):
raise EncryptionError("Error decoding key.")
try:
# Drop last 8 bytes, which are 0
iv = unpaddedbase64.decode_base64(iv)[:8]
except (base64.binascii.Error, TypeError):
raise EncryptionError("Error decoding initial values.")
ctr = Counter.new(64, prefix=iv, initial_value=0)
try:
cipher = AES.new(key, AES.MODE_CTR, counter=ctr)
except ValueError as e:
raise EncryptionError(e)
return cipher.decrypt(ciphertext)
def verifyServerSignedJson(self,
signed_json,
acceptable_server_names=None):
"""Given a signed json object, try to verify any one
of the signatures on it
XXX: This contains a fairly noddy version of the home server
SRV lookup and signature verification. It does no caching (just
fetches the signature each time and does not contact any other
servers to do perspective checks).
:param acceptable_server_names: If provided and not None,
only signatures from servers in this list will be accepted.
:type acceptable_server_names: list[unicode] or None
:return a tuple of the server name and key name that was
successfully verified.
:rtype: twisted.internet.defer.Deferred[tuple[unicode]]
:raise SignatureVerifyException: The json cannot be verified.
"""
if 'signatures' not in signed_json:
raise SignatureVerifyException("Signature missing")
for server_name, sigs in signed_json['signatures'].items():
if acceptable_server_names is not None:
if server_name not in acceptable_server_names:
continue
server_keys = yield self._getKeysForServer(server_name)
for key_name, sig in sigs.items():
if key_name in server_keys:
if 'key' not in server_keys[key_name]:
logger.warn("Ignoring key %s with no 'key'")
continue
key_bytes = decode_base64(server_keys[key_name]['key'])
verify_key = signedjson.key.decode_verify_key_bytes(
key_name, key_bytes)
logger.info("verifying sig from key %r", key_name)
signedjson.sign.verify_signed_json(signed_json,
server_name, verify_key)
logger.info("Verified signature with key %s from %s",
key_name, server_name)
defer.returnValue((server_name, key_name))
logger.warn(
"No matching key found for signature block %r in server keys %r",
signed_json['signatures'],
server_keys,
)
logger.warn(
"Unable to verify any signatures from block %r. Acceptable server names: %r",
signed_json['signatures'],
acceptable_server_names,
)
raise SignatureVerifyException("No matching signature found")
def decode_verify_key_base64(algorithm, version, key_base64):
"""Decode a base64 encoded verify key
Args:
algorithm (str): The algorithm the key is for (currently "ed25519").
version (str): Identifies this key out of the keys for this entity.
key_base64 (str): Base64 encoded bytes of the key.
Returns:
A VerifyKey object.
"""
key_id = "%s:%s" % (algorithm, version)
key_bytes = decode_base64(key_base64)
return decode_verify_key_bytes(key_id, key_bytes)
def test_decode(self):
self.assertEqual(decode_base64(u''), b'')
self.assertEqual(decode_base64(u'AA'), b'\x00')
self.assertEqual(decode_base64(u'AAA'), b'\x00\x00')
self.assertEqual(decode_base64(u'AAAA'), b'\x00\x00\x00')
with self.assertRaises(Exception):
decode_base64(u'A')
def test_decode(self) -> None:
self.assertEqual(decode_base64(""), b"")
self.assertEqual(decode_base64("AA"), b"\x00")
self.assertEqual(decode_base64("AAA"), b"\x00\x00")
self.assertEqual(decode_base64("AAAA"), b"\x00\x00\x00")
with self.assertRaises(Exception):
decode_base64("A")
def test_decode(self):
self.assertEqual(decode_base64(u''), b'')
self.assertEqual(decode_base64(u'AA'), b'\x00')
self.assertEqual(decode_base64(u'AAA'), b'\x00\x00')
self.assertEqual(decode_base64(u'AAAA'), b'\x00\x00\x00')
with self.assertRaises(Exception):
decode_base64(u'A')
async def verifyServerSignedJson(
self,
signed_json: SignedMatrixRequest,
acceptable_server_names: Optional[List[str]] = None,
) -> Tuple[str, str]:
"""Given a signed json object, try to verify any one
of the signatures on it
XXX: This contains a fairly noddy version of the home server
SRV lookup and signature verification. It does no caching (just
fetches the signature each time and does not contact any other
servers to do perspective checks).
:param acceptable_server_names: If provided and not None,
only signatures from servers in this list will be accepted.
:return a tuple of the server name and key name that was
successfully verified.
:raise SignatureVerifyException: The json cannot be verified.
"""
for server_name, sigs in signed_json.signatures.items():
if acceptable_server_names is not None:
if server_name not in acceptable_server_names:
continue
server_keys = await self._getKeysForServer(server_name)
for key_name, sig in sigs.items():
if key_name in server_keys:
key_bytes = decode_base64(server_keys[key_name]["key"])
verify_key = signedjson.key.decode_verify_key_bytes(
key_name, key_bytes)
logger.info("verifying sig from key %r", key_name)
payload = attr.asdict(signed_json)
signedjson.sign.verify_signed_json(payload, server_name,
verify_key)
logger.info("Verified signature with key %s from %s",
key_name, server_name)
return (server_name, key_name)
logger.warning(
"No matching key found for signature block %r in server keys %r",
signed_json.signatures,
server_keys,
)
logger.warning(
"Unable to verify any signatures from block %r. Acceptable server names: %r",
signed_json.signatures,
acceptable_server_names,
)
raise SignatureVerifyException("No matching signature found")
def read_old_signing_keys(self, old_signing_keys):
keys = {}
for key_id, key_data in old_signing_keys.items():
if is_signing_algorithm_supported(key_id):
key_base64 = key_data["key"]
key_bytes = decode_base64(key_base64)
verify_key = decode_verify_key_bytes(key_id, key_bytes)
verify_key.expired_ts = key_data["expired_ts"]
keys[key_id] = verify_key
else:
raise ConfigError(
"Unsupported signing algorithm for old key: %r" % (key_id,)
)
return keys
def __init__(self, sydent, server_name, port, pubkeys):
super(RemotePeer, self).__init__(server_name, pubkeys)
self.sydent = sydent
# look up or build the replication URL
try:
replication_url = sydent.cfg.get(
"peer.%s" % server_name,
"base_replication_url",
)
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
if not port:
port = 1001
replication_url = "https://%s:%i" % (server_name, port)
if replication_url[-1:] != '/':
replication_url += "/"
replication_url += "_matrix/identity/replicate/v1/push"
self.replication_url = replication_url
# Get verify key for this peer
# Check if their key is base64 or hex encoded
pubkey = self.pubkeys[SIGNING_KEY_ALGORITHM]
try:
# Check for hex encoding
int(pubkey, 16)
# Decode hex into bytes
pubkey_decoded = binascii.unhexlify(pubkey)
logger.warn(
"Peer public key of %s is hex encoded. Please update to base64 encoding",
server_name)
except ValueError:
# Check for base64 encoding
try:
pubkey_decoded = decode_base64(pubkey)
except Exception as e:
raise ConfigError(
"Unable to decode public key for peer %s: %s" %
(server_name, e), )
self.verify_key = signedjson.key.decode_verify_key_bytes(
SIGNING_KEY_ALGORITHM + ":", pubkey_decoded)
# Attach metadata
self.verify_key.alg = SIGNING_KEY_ALGORITHM
self.verify_key.version = 0
def verify_signed_json(json_object, signature_name, verify_key):
# type: (JsonDict, str, VerifyKey) -> None
"""Check a signature on a signed JSON object.
Args:
json_object: The signed JSON object to check.
signature_name: The name of the signature to check.
verify_key: The key to verify the signature.
Raises:
SignatureVerifyException: If the signature isn't valid
"""
try:
signatures = json_object["signatures"]
except KeyError:
raise SignatureVerifyException("No signatures on this object")
key_id = "%s:%s" % (verify_key.alg, verify_key.version)
try:
signature_b64 = signatures[signature_name][key_id]
except KeyError:
raise SignatureVerifyException("Missing signature for %s, %s" %
(signature_name, key_id))
try:
signature = decode_base64(signature_b64)
except Exception:
raise SignatureVerifyException("Invalid signature base64 for %s, %s" %
(signature_name, key_id))
json_object_copy = dict(json_object)
del json_object_copy["signatures"]
json_object_copy.pop("unsigned", None)
message = encode_canonical_json(json_object_copy)
# logger.debug("VERIFY: %s %s %s", signature_name, key_id, message)
try:
verify_key.verify(message, signature)
except Exception as e:
raise SignatureVerifyException(
"Unable to verify signature for %s: %s %s" % (
signature_name,
type(e),
e,
))
def read_old_signing_keys(stream):
"""Reads a list of old keys from a stream
Args:
stream : A stream to iterate for keys.
Returns:
list of VerifyKey objects.
"""
keys = []
for line in stream:
algorithm, version, expired, key_base64 = line.split()
key_name = "%s:%s" % (algorithm, version,)
key = decode_verify_key_bytes(key_name, decode_base64(key_base64))
key.expired = int(expired)
keys.append(key)
return keys
def decode_signing_key_base64(algorithm, version, key_base64):
"""Decode a base64 encoded signing key
Args:
algorithm (str): The algorithm the key is for (currently "ed25519").
version (str): Identifies this key out of the keys for this entity.
key_base64 (str): Base64 encoded bytes of the key.
Returns:
A SigningKey object.
"""
if algorithm == NACL_ED25519:
key_bytes = decode_base64(key_base64)
key = nacl.signing.SigningKey(key_bytes)
key.version = version
key.alg = NACL_ED25519
return key
else:
raise ValueError("Unsupported algorithm %s" % (algorithm,))
def _verify_any_signature(self, data, server_hostname):
if server_hostname not in data["signatures"]:
raise AuthError(401, "No signature from server %s" % (server_hostname,))
for key_name, signature in data["signatures"][server_hostname].items():
key_data = yield self.simple_http_client.get_json(
"%s%s/_matrix/identity/api/v1/pubkey/%s" %
(id_server_scheme, server_hostname, key_name,),
)
if "public_key" not in key_data:
raise AuthError(401, "No public key named %s from %s" %
(key_name, server_hostname,))
verify_signed_json(
data,
server_hostname,
decode_verify_key_bytes(key_name, decode_base64(key_data["public_key"]))
)
return
def __init__(self, sydent, server_name, port, pubkeys):
super(RemotePeer, self).__init__(server_name, pubkeys)
self.sydent = sydent
# look up or build the replication URL
try:
replication_url = sydent.cfg.get(
"peer.%s" % server_name, "base_replication_url",
)
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
if not port:
port = 1001
replication_url = "https://%s:%i" % (server_name, port)
if replication_url[-1:] != '/':
replication_url += "/"
replication_url += "_matrix/identity/replicate/v1/push"
self.replication_url = replication_url
# Get verify key for this peer
# Check if their key is base64 or hex encoded
pubkey = self.pubkeys[SIGNING_KEY_ALGORITHM]
try:
# Check for hex encoding
int(pubkey, 16)
# Decode hex into bytes
pubkey_decoded = binascii.unhexlify(pubkey)
logger.warn("Peer public key of %s is hex encoded. Please update to base64 encoding", server_name)
except ValueError:
# Check for base64 encoding
try:
pubkey_decoded = decode_base64(pubkey)
except Exception as e:
raise ConfigError(
"Unable to decode public key for peer %s: %s" % (server_name, e),
)
self.verify_key = signedjson.key.decode_verify_key_bytes(SIGNING_KEY_ALGORITHM + ":", pubkey_decoded)
# Attach metadata
self.verify_key.alg = SIGNING_KEY_ALGORITHM
self.verify_key.version = 0
def verify_any_signature(self, data, server_hostname):
if server_hostname not in data["signatures"]:
raise AuthError(401, "No signature from server %s" % (server_hostname,))
for key_name, signature in data["signatures"][server_hostname].items():
key_data = yield self.hs.get_simple_http_client().get_json(
"%s%s/_matrix/identity/api/v1/pubkey/%s" %
(id_server_scheme, server_hostname, key_name,),
)
if "public_key" not in key_data:
raise AuthError(401, "No public key named %s from %s" %
(key_name, server_hostname,))
verify_signed_json(
data,
server_hostname,
decode_verify_key_bytes(key_name, decode_base64(key_data["public_key"]))
)
return
def decode_signing_key_base64(algorithm, version, key_base64):
"""Decode a base64 encoded signing key
Args:
algorithm (str): The algorithm the key is for (currently "ed25519").
version (str): Identifies this key out of the keys for this entity.
key_base64 (str): Base64 encoded bytes of the key.
Returns:
A SigningKey object.
"""
if algorithm == NACL_ED25519:
key_bytes = decode_base64(key_base64)
key = nacl.signing.SigningKey(key_bytes)
key.version = version
key.alg = NACL_ED25519
return key
else:
raise ValueError("Unsupported algorithm %s" % (algorithm, ))
def verifyServerSignedJson(self, signed_json, acceptable_server_names=None):
"""Given a signed json object, try to verify any one
of the signatures on it
XXX: This contains a fairly noddy version of the home server
SRV lookup and signature verification. It does no caching (just
fetches the signature each time and does not contact any other
servers to do perspective checks).
:param acceptable_server_names: If provided and not None,
only signatures from servers in this list will be accepted.
:type acceptable_server_names: list of strings
:return a tuple of the server name and key name that was
successfully verified. If the json cannot be verified,
raises SignatureVerifyException.
"""
if 'signatures' not in signed_json:
raise SignatureVerifyException("Signature missing")
for server_name, sigs in signed_json['signatures'].items():
if acceptable_server_names is not None:
if server_name not in acceptable_server_names:
continue
server_keys = yield self._getKeysForServer(server_name)
for key_name, sig in sigs.items():
if key_name in server_keys:
if 'key' not in server_keys[key_name]:
logger.warn("Ignoring key %s with no 'key'")
continue
key_bytes = decode_base64(server_keys[key_name]['key'])
verify_key = signedjson.key.decode_verify_key_bytes(key_name, key_bytes)
logger.info("verifying sig from key %r", key_name)
signedjson.sign.verify_signed_json(signed_json, server_name, verify_key)
logger.info("Verified signature with key %s from %s", key_name, server_name)
defer.returnValue((server_name, key_name))
logger.warn(
"No matching key found for signature block %r in server keys %r",
signed_json['signatures'], server_keys,
)
logger.warn(
"Unable to verify any signatures from block %r. Acceptable server names: %r",
signed_json['signatures'], acceptable_server_names,
)
raise SignatureVerifyException("No matching signature found")
def read_old_signing_keys(
self, old_signing_keys: Optional[JsonDict]
) -> Dict[str, "VerifyKeyWithExpiry"]:
if old_signing_keys is None:
return {}
keys = {}
for key_id, key_data in old_signing_keys.items():
if is_signing_algorithm_supported(key_id):
key_base64 = key_data["key"]
key_bytes = decode_base64(key_base64)
verify_key: "VerifyKeyWithExpiry" = decode_verify_key_bytes(
key_id, key_bytes) # type: ignore[assignment]
verify_key.expired = key_data["expired_ts"]
keys[key_id] = verify_key
else:
raise ConfigError(
"Unsupported signing algorithm for old key: %r" %
(key_id, ))
return keys
def get_verify_key_from_cross_signing_key(key_info):
"""Get the key ID and signedjson verify key from a cross-signing key dict
Args:
key_info (dict): a cross-signing key dict, which must have a "keys"
property that has exactly one item in it
Returns:
(str, VerifyKey): the key ID and verify key for the cross-signing key
"""
# make sure that exactly one key is provided
if "keys" not in key_info:
raise ValueError("Invalid key")
keys = key_info["keys"]
if len(keys) != 1:
raise ValueError("Invalid key")
# and return that one key
for key_id, key_data in keys.items():
return key_id, decode_verify_key_bytes(key_id, decode_base64(key_data))
def test_encrypt_decrypt():
plaintext = b'test'
passphrase = 'pass'
# Set a ridiculously low round count for this test to be fast
ciphertext = encrypt(plaintext, passphrase, count=1)
assert decrypt(ciphertext, passphrase) == plaintext
ciphertext_bytes = decode_base64(ciphertext)
# Wrong hmac
ciphertext = encode_base64(ciphertext_bytes[:-32] + b'A' * 32)
with pytest.raises(ValueError):
decrypt(ciphertext, passphrase)
# Wrong version
ciphertext = encode_base64(bytes([42]) + ciphertext_bytes[1:])
with pytest.raises(ValueError):
decrypt(ciphertext, passphrase)
def _check_master_key_signature(
self, user_id, master_key_id, signed_master_key, stored_master_key, devices
):
"""Check signatures of a user's master key made by their devices.
Args:
user_id (string): the user whose master key is being checked
master_key_id (string): the ID of the user's master key
signed_master_key (dict): the user's signed master key that was uploaded
stored_master_key (dict): our previously-stored copy of the user's master key
devices (iterable(dict)): the user's devices
Returns:
list[SignatureListItem]: a list of signatures to store
Raises:
SynapseError: if a signature is invalid
"""
# for each device that signed the master key, check the signature.
master_key_signature_list = []
sigs = signed_master_key["signatures"]
for signing_key_id, signature in sigs[user_id].items():
_, signing_device_id = signing_key_id.split(":", 1)
if (
signing_device_id not in devices
or signing_key_id not in devices[signing_device_id]["keys"]
):
# signed by an unknown device, or the
# device does not have the key
raise SynapseError(400, "Invalid signature", Codes.INVALID_SIGNATURE)
# get the key and check the signature
pubkey = devices[signing_device_id]["keys"][signing_key_id]
verify_key = decode_verify_key_bytes(signing_key_id, decode_base64(pubkey))
_check_device_signature(
user_id, verify_key, signed_master_key, stored_master_key
)
master_key_signature_list.append(
SignatureListItem(signing_key_id, user_id, master_key_id, signature)
)
return master_key_signature_list
def verify_signed_json(json_object, signature_name, verify_key):
"""Check a signature on a signed JSON object.
Args:
json_object (dict): The signed JSON object to check.
signature_name (str): The name of the signature to check.
verify_key (syutil.crypto.VerifyKey): The key to verify the signature.
Raises:
InvalidSignature: If the signature isn't valid
"""
try:
signatures = json_object["signatures"]
except KeyError:
raise SignatureVerifyException("No signatures on this object")
key_id = "%s:%s" % (verify_key.alg, verify_key.version)
try:
signature_b64 = signatures[signature_name][key_id]
except:
raise SignatureVerifyException("Missing signature for %s, %s" % (signature_name, key_id))
try:
signature = decode_base64(signature_b64)
except:
raise SignatureVerifyException("Invalid signature base64 for %s, %s" % (signature_name, key_id))
json_object_copy = dict(json_object)
del json_object_copy["signatures"]
json_object_copy.pop("unsigned", None)
message = encode_canonical_json(json_object_copy)
# logger.debug("VERIFY: %s %s %s", signature_name, key_id, message)
try:
verify_key.verify(message, signature)
except:
logger.exception("Error verifying signature")
raise SignatureVerifyException("Unable to verify signature for %s " % signature_name)
def get_verify_key_from_cross_signing_key(
key_info: Mapping[str, Any]) -> Tuple[str, VerifyKey]:
"""Get the key ID and signedjson verify key from a cross-signing key dict
Args:
key_info: a cross-signing key dict, which must have a "keys"
property that has exactly one item in it
Returns:
the key ID and verify key for the cross-signing key
"""
# make sure that a `keys` field is provided
if "keys" not in key_info:
raise ValueError("Invalid key")
keys = key_info["keys"]
# and that it contains exactly one key
if len(keys) == 1:
key_id, key_data = next(iter(keys.items()))
return key_id, decode_verify_key_bytes(key_id, decode_base64(key_data))
else:
raise ValueError("Invalid key")
def _verify_any_signature(self, data, server_hostname):
if server_hostname not in data["signatures"]:
raise AuthError(
401, "No signature from server %s" % (server_hostname, ))
for key_name, signature in data["signatures"][server_hostname].items():
try:
key_data = yield self.blacklisting_http_client.get_json(
"%s%s/_matrix/identity/api/v1/pubkey/%s" %
(id_server_scheme, server_hostname, key_name))
except TimeoutError:
raise SynapseError(500, "Timed out contacting identity server")
if "public_key" not in key_data:
raise AuthError(
401, "No public key named %s from %s" %
(key_name, server_hostname))
verify_signed_json(
data,
server_hostname,
decode_verify_key_bytes(key_name,
decode_base64(key_data["public_key"])),
)
return
def check_event_content_hash(event, hash_algorithm=hashlib.sha256):
"""Check whether the hash for this PDU matches the contents"""
name, expected_hash = compute_content_hash(event, hash_algorithm)
logger.debug("Expecting hash: %s", encode_base64(expected_hash))
if name not in event.hashes:
raise SynapseError(
400,
"Algorithm %s not in hashes %s" % (
name, list(event.hashes),
),
Codes.UNAUTHORIZED,
)
message_hash_base64 = event.hashes[name]
try:
message_hash_bytes = decode_base64(message_hash_base64)
except:
raise SynapseError(
400,
"Invalid base64: %s" % (message_hash_base64,),
Codes.UNAUTHORIZED,
)
return message_hash_bytes == expected_hash
def process_v2_response(self, from_server, response_json,
requested_ids=[], only_from_server=True):
time_now_ms = self.clock.time_msec()
response_keys = {}
verify_keys = {}
for key_id, key_data in response_json["verify_keys"].items():
if is_signing_algorithm_supported(key_id):
key_base64 = key_data["key"]
key_bytes = decode_base64(key_base64)
verify_key = decode_verify_key_bytes(key_id, key_bytes)
verify_key.time_added = time_now_ms
verify_keys[key_id] = verify_key
old_verify_keys = {}
for key_id, key_data in response_json["old_verify_keys"].items():
if is_signing_algorithm_supported(key_id):
key_base64 = key_data["key"]
key_bytes = decode_base64(key_base64)
verify_key = decode_verify_key_bytes(key_id, key_bytes)
verify_key.expired = key_data["expired_ts"]
verify_key.time_added = time_now_ms
old_verify_keys[key_id] = verify_key
results = {}
server_name = response_json["server_name"]
if only_from_server:
if server_name != from_server:
raise ValueError(
"Expected a response for server %r not %r" % (
from_server, server_name
)
)
for key_id in response_json["signatures"].get(server_name, {}):
if key_id not in response_json["verify_keys"]:
raise ValueError(
"Key response must include verification keys for all"
" signatures"
)
if key_id in verify_keys:
verify_signed_json(
response_json,
server_name,
verify_keys[key_id]
)
signed_key_json = sign_json(
response_json,
self.config.server_name,
self.config.signing_key[0],
)
signed_key_json_bytes = encode_canonical_json(signed_key_json)
ts_valid_until_ms = signed_key_json[u"valid_until_ts"]
updated_key_ids = set(requested_ids)
updated_key_ids.update(verify_keys)
updated_key_ids.update(old_verify_keys)
response_keys.update(verify_keys)
response_keys.update(old_verify_keys)
yield defer.gatherResults(
[
preserve_fn(self.store.store_server_keys_json)(
server_name=server_name,
key_id=key_id,
from_server=server_name,
ts_now_ms=time_now_ms,
ts_expires_ms=ts_valid_until_ms,
key_json_bytes=signed_key_json_bytes,
)
for key_id in updated_key_ids
],
consumeErrors=True,
).addErrback(unwrapFirstError)
results[server_name] = response_keys
defer.returnValue(results)
def search(self, user, content, batch=None):
"""Performs a full text search for a user.
Args:
user (UserID)
content (dict): Search parameters
batch (str): The next_batch parameter. Used for pagination.
Returns:
dict to be returned to the client with results of search
"""
if not self.hs.config.enable_search:
raise SynapseError(400, "Search is disabled on this homeserver")
batch_group = None
batch_group_key = None
batch_token = None
if batch:
try:
b = decode_base64(batch).decode('ascii')
batch_group, batch_group_key, batch_token = b.split("\n")
assert batch_group is not None
assert batch_group_key is not None
assert batch_token is not None
except Exception:
raise SynapseError(400, "Invalid batch")
logger.info(
"Search batch properties: %r, %r, %r",
batch_group, batch_group_key, batch_token,
)
logger.info("Search content: %s", content)
try:
room_cat = content["search_categories"]["room_events"]
# The actual thing to query in FTS
search_term = room_cat["search_term"]
# Which "keys" to search over in FTS query
keys = room_cat.get("keys", [
"content.body", "content.name", "content.topic",
])
# Filter to apply to results
filter_dict = room_cat.get("filter", {})
# What to order results by (impacts whether pagination can be doen)
order_by = room_cat.get("order_by", "rank")
# Return the current state of the rooms?
include_state = room_cat.get("include_state", False)
# Include context around each event?
event_context = room_cat.get(
"event_context", None
)
# Group results together? May allow clients to paginate within a
# group
group_by = room_cat.get("groupings", {}).get("group_by", {})
group_keys = [g["key"] for g in group_by]
if event_context is not None:
before_limit = int(event_context.get(
"before_limit", 5
))
after_limit = int(event_context.get(
"after_limit", 5
))
# Return the historic display name and avatar for the senders
# of the events?
include_profile = bool(event_context.get("include_profile", False))
except KeyError:
raise SynapseError(400, "Invalid search query")
if order_by not in ("rank", "recent"):
raise SynapseError(400, "Invalid order by: %r" % (order_by,))
if set(group_keys) - {"room_id", "sender"}:
raise SynapseError(
400,
"Invalid group by keys: %r" % (set(group_keys) - {"room_id", "sender"},)
)
search_filter = Filter(filter_dict)
# TODO: Search through left rooms too
rooms = yield self.store.get_rooms_for_user_where_membership_is(
user.to_string(),
membership_list=[Membership.JOIN],
# membership_list=[Membership.JOIN, Membership.LEAVE, Membership.Ban],
)
room_ids = set(r.room_id for r in rooms)
# If doing a subset of all rooms seearch, check if any of the rooms
# are from an upgraded room, and search their contents as well
if search_filter.rooms:
historical_room_ids = []
for room_id in search_filter.rooms:
# Add any previous rooms to the search if they exist
ids = yield self.get_old_rooms_from_upgraded_room(room_id)
historical_room_ids += ids
# Prevent any historical events from being filtered
search_filter = search_filter.with_room_ids(historical_room_ids)
room_ids = search_filter.filter_rooms(room_ids)
if batch_group == "room_id":
room_ids.intersection_update({batch_group_key})
if not room_ids:
defer.returnValue({
"search_categories": {
"room_events": {
"results": [],
"count": 0,
"highlights": [],
}
}
})
rank_map = {} # event_id -> rank of event
allowed_events = []
room_groups = {} # Holds result of grouping by room, if applicable
sender_group = {} # Holds result of grouping by sender, if applicable
# Holds the next_batch for the entire result set if one of those exists
global_next_batch = None
highlights = set()
count = None
if order_by == "rank":
search_result = yield self.store.search_msgs(
room_ids, search_term, keys
)
count = search_result["count"]
if search_result["highlights"]:
highlights.update(search_result["highlights"])
results = search_result["results"]
results_map = {r["event"].event_id: r for r in results}
rank_map.update({r["event"].event_id: r["rank"] for r in results})
filtered_events = search_filter.filter([r["event"] for r in results])
events = yield filter_events_for_client(
self.store, user.to_string(), filtered_events
)
events.sort(key=lambda e: -rank_map[e.event_id])
allowed_events = events[:search_filter.limit()]
for e in allowed_events:
rm = room_groups.setdefault(e.room_id, {
"results": [],
"order": rank_map[e.event_id],
})
rm["results"].append(e.event_id)
s = sender_group.setdefault(e.sender, {
"results": [],
"order": rank_map[e.event_id],
})
s["results"].append(e.event_id)
elif order_by == "recent":
room_events = []
i = 0
pagination_token = batch_token
# We keep looping and we keep filtering until we reach the limit
# or we run out of things.
# But only go around 5 times since otherwise synapse will be sad.
while len(room_events) < search_filter.limit() and i < 5:
i += 1
search_result = yield self.store.search_rooms(
room_ids, search_term, keys, search_filter.limit() * 2,
pagination_token=pagination_token,
)
if search_result["highlights"]:
highlights.update(search_result["highlights"])
count = search_result["count"]
results = search_result["results"]
results_map = {r["event"].event_id: r for r in results}
rank_map.update({r["event"].event_id: r["rank"] for r in results})
filtered_events = search_filter.filter([
r["event"] for r in results
])
events = yield filter_events_for_client(
self.store, user.to_string(), filtered_events
)
room_events.extend(events)
room_events = room_events[:search_filter.limit()]
if len(results) < search_filter.limit() * 2:
pagination_token = None
break
else:
pagination_token = results[-1]["pagination_token"]
for event in room_events:
group = room_groups.setdefault(event.room_id, {
"results": [],
})
group["results"].append(event.event_id)
if room_events and len(room_events) >= search_filter.limit():
last_event_id = room_events[-1].event_id
pagination_token = results_map[last_event_id]["pagination_token"]
# We want to respect the given batch group and group keys so
# that if people blindly use the top level `next_batch` token
# it returns more from the same group (if applicable) rather
# than reverting to searching all results again.
if batch_group and batch_group_key:
global_next_batch = encode_base64(("%s\n%s\n%s" % (
batch_group, batch_group_key, pagination_token
)).encode('ascii'))
else:
global_next_batch = encode_base64(("%s\n%s\n%s" % (
"all", "", pagination_token
)).encode('ascii'))
for room_id, group in room_groups.items():
group["next_batch"] = encode_base64(("%s\n%s\n%s" % (
"room_id", room_id, pagination_token
)).encode('ascii'))
allowed_events.extend(room_events)
else:
# We should never get here due to the guard earlier.
raise NotImplementedError()
logger.info("Found %d events to return", len(allowed_events))
# If client has asked for "context" for each event (i.e. some surrounding
# events and state), fetch that
if event_context is not None:
now_token = yield self.hs.get_event_sources().get_current_token()
contexts = {}
for event in allowed_events:
res = yield self.store.get_events_around(
event.room_id, event.event_id, before_limit, after_limit,
)
logger.info(
"Context for search returned %d and %d events",
len(res["events_before"]), len(res["events_after"]),
)
res["events_before"] = yield filter_events_for_client(
self.store, user.to_string(), res["events_before"]
)
res["events_after"] = yield filter_events_for_client(
self.store, user.to_string(), res["events_after"]
)
res["start"] = now_token.copy_and_replace(
"room_key", res["start"]
).to_string()
res["end"] = now_token.copy_and_replace(
"room_key", res["end"]
).to_string()
if include_profile:
senders = set(
ev.sender
for ev in itertools.chain(
res["events_before"], [event], res["events_after"]
)
)
if res["events_after"]:
last_event_id = res["events_after"][-1].event_id
else:
last_event_id = event.event_id
state_filter = StateFilter.from_types(
[(EventTypes.Member, sender) for sender in senders]
)
state = yield self.store.get_state_for_event(
last_event_id, state_filter
)
res["profile_info"] = {
s.state_key: {
"displayname": s.content.get("displayname", None),
"avatar_url": s.content.get("avatar_url", None),
}
for s in state.values()
if s.type == EventTypes.Member and s.state_key in senders
}
contexts[event.event_id] = res
else:
contexts = {}
# TODO: Add a limit
time_now = self.clock.time_msec()
for context in contexts.values():
context["events_before"] = (
yield self._event_serializer.serialize_events(
context["events_before"], time_now,
)
)
context["events_after"] = (
yield self._event_serializer.serialize_events(
context["events_after"], time_now,
)
)
state_results = {}
if include_state:
rooms = set(e.room_id for e in allowed_events)
for room_id in rooms:
state = yield self.state_handler.get_current_state(room_id)
state_results[room_id] = list(state.values())
state_results.values()
# We're now about to serialize the events. We should not make any
# blocking calls after this. Otherwise the 'age' will be wrong
results = []
for e in allowed_events:
results.append({
"rank": rank_map[e.event_id],
"result": (yield self._event_serializer.serialize_event(e, time_now)),
"context": contexts.get(e.event_id, {}),
})
rooms_cat_res = {
"results": results,
"count": count,
"highlights": list(highlights),
}
if state_results:
s = {}
for room_id, state in state_results.items():
s[room_id] = yield self._event_serializer.serialize_events(
state, time_now,
)
rooms_cat_res["state"] = s
if room_groups and "room_id" in group_keys:
rooms_cat_res.setdefault("groups", {})["room_id"] = room_groups
if sender_group and "sender" in group_keys:
rooms_cat_res.setdefault("groups", {})["sender"] = sender_group
if global_next_batch:
rooms_cat_res["next_batch"] = global_next_batch
defer.returnValue({
"search_categories": {
"room_events": rooms_cat_res
}
})
def get_server_verify_key_v1_direct(self, server_name, key_ids):
"""Finds a verification key for the server with one of the key ids.
Args:
server_name (str): The name of the server to fetch a key for.
keys_ids (list of str): The key_ids to check for.
"""
# Try to fetch the key from the remote server.
(response, tls_certificate) = yield fetch_server_key(
server_name, self.hs.tls_server_context_factory
)
# Check the response.
x509_certificate_bytes = crypto.dump_certificate(
crypto.FILETYPE_ASN1, tls_certificate
)
if ("signatures" not in response
or server_name not in response["signatures"]):
raise ValueError("Key response not signed by remote server")
if "tls_certificate" not in response:
raise ValueError("Key response missing TLS certificate")
tls_certificate_b64 = response["tls_certificate"]
if encode_base64(x509_certificate_bytes) != tls_certificate_b64:
raise ValueError("TLS certificate doesn't match")
# Cache the result in the datastore.
time_now_ms = self.clock.time_msec()
verify_keys = {}
for key_id, key_base64 in response["verify_keys"].items():
if is_signing_algorithm_supported(key_id):
key_bytes = decode_base64(key_base64)
verify_key = decode_verify_key_bytes(key_id, key_bytes)
verify_key.time_added = time_now_ms
verify_keys[key_id] = verify_key
for key_id in response["signatures"][server_name]:
if key_id not in response["verify_keys"]:
raise ValueError(
"Key response must include verification keys for all"
" signatures"
)
if key_id in verify_keys:
verify_signed_json(
response,
server_name,
verify_keys[key_id]
)
yield self.store.store_server_certificate(
server_name,
server_name,
time_now_ms,
tls_certificate,
)
yield self.store_keys(
server_name=server_name,
from_server=server_name,
verify_keys=verify_keys,
)
defer.returnValue(verify_keys)
def test_decode_urlunsafe_chars(self):
self.assertEqual(decode_base64(u'/+aa'), b'\xff\xe6\x9a')
self.assertEqual(decode_base64(u'_-aa'), b'\xff\xe6\x9a')
def select_pdus(cursor):
cursor.execute(
"SELECT pdu_id, origin FROM pdus ORDER BY depth ASC"
)
ids = cursor.fetchall()
pdu_tuples = store._get_pdu_tuples(cursor, ids)
pdus = [Pdu.from_pdu_tuple(p) for p in pdu_tuples]
reference_hashes = {}
for pdu in pdus:
try:
if pdu.prev_pdus:
print "PROCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus
for pdu_id, origin, hashes in pdu.prev_pdus:
ref_alg, ref_hsh = reference_hashes[(pdu_id, origin)]
hashes[ref_alg] = encode_base64(ref_hsh)
store._store_prev_pdu_hash_txn(cursor, pdu.pdu_id, pdu.origin, pdu_id, origin, ref_alg, ref_hsh)
print "SUCCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus
pdu = add_event_pdu_content_hash(pdu)
ref_alg, ref_hsh = compute_pdu_event_reference_hash(pdu)
reference_hashes[(pdu.pdu_id, pdu.origin)] = (ref_alg, ref_hsh)
store._store_pdu_reference_hash_txn(cursor, pdu.pdu_id, pdu.origin, ref_alg, ref_hsh)
for alg, hsh_base64 in pdu.hashes.items():
print alg, hsh_base64
store._store_pdu_content_hash_txn(cursor, pdu.pdu_id, pdu.origin, alg, decode_base64(hsh_base64))
except:
print "FAILED_", pdu.pdu_id, pdu.origin, pdu.prev_pdus
def search(self, user, content, batch=None):
"""Performs a full text search for a user.
Args:
user (UserID)
content (dict): Search parameters
batch (str): The next_batch parameter. Used for pagination.
Returns:
dict to be returned to the client with results of search
"""
batch_group = None
batch_group_key = None
batch_token = None
if batch:
try:
b = decode_base64(batch)
batch_group, batch_group_key, batch_token = b.split("\n")
assert batch_group is not None
assert batch_group_key is not None
assert batch_token is not None
except:
raise SynapseError(400, "Invalid batch")
try:
room_cat = content["search_categories"]["room_events"]
# The actual thing to query in FTS
search_term = room_cat["search_term"]
# Which "keys" to search over in FTS query
keys = room_cat.get("keys", [
"content.body", "content.name", "content.topic",
])
# Filter to apply to results
filter_dict = room_cat.get("filter", {})
# What to order results by (impacts whether pagination can be doen)
order_by = room_cat.get("order_by", "rank")
# Return the current state of the rooms?
include_state = room_cat.get("include_state", False)
# Include context around each event?
event_context = room_cat.get(
"event_context", None
)
# Group results together? May allow clients to paginate within a
# group
group_by = room_cat.get("groupings", {}).get("group_by", {})
group_keys = [g["key"] for g in group_by]
if event_context is not None:
before_limit = int(event_context.get(
"before_limit", 5
))
after_limit = int(event_context.get(
"after_limit", 5
))
# Return the historic display name and avatar for the senders
# of the events?
include_profile = bool(event_context.get("include_profile", False))
except KeyError:
raise SynapseError(400, "Invalid search query")
if order_by not in ("rank", "recent"):
raise SynapseError(400, "Invalid order by: %r" % (order_by,))
if set(group_keys) - {"room_id", "sender"}:
raise SynapseError(
400,
"Invalid group by keys: %r" % (set(group_keys) - {"room_id", "sender"},)
)
search_filter = Filter(filter_dict)
# TODO: Search through left rooms too
rooms = yield self.store.get_rooms_for_user_where_membership_is(
user.to_string(),
membership_list=[Membership.JOIN],
# membership_list=[Membership.JOIN, Membership.LEAVE, Membership.Ban],
)
room_ids = set(r.room_id for r in rooms)
room_ids = search_filter.filter_rooms(room_ids)
if batch_group == "room_id":
room_ids.intersection_update({batch_group_key})
rank_map = {} # event_id -> rank of event
allowed_events = []
room_groups = {} # Holds result of grouping by room, if applicable
sender_group = {} # Holds result of grouping by sender, if applicable
# Holds the next_batch for the entire result set if one of those exists
global_next_batch = None
if order_by == "rank":
results = yield self.store.search_msgs(
room_ids, search_term, keys
)
results_map = {r["event"].event_id: r for r in results}
rank_map.update({r["event"].event_id: r["rank"] for r in results})
filtered_events = search_filter.filter([r["event"] for r in results])
events = yield self._filter_events_for_client(
user.to_string(), filtered_events
)
events.sort(key=lambda e: -rank_map[e.event_id])
allowed_events = events[:search_filter.limit()]
for e in allowed_events:
rm = room_groups.setdefault(e.room_id, {
"results": [],
"order": rank_map[e.event_id],
})
rm["results"].append(e.event_id)
s = sender_group.setdefault(e.sender, {
"results": [],
"order": rank_map[e.event_id],
})
s["results"].append(e.event_id)
elif order_by == "recent":
# In this case we specifically loop through each room as the given
# limit applies to each room, rather than a global list.
# This is not necessarilly a good idea.
for room_id in room_ids:
room_events = []
if batch_group == "room_id" and batch_group_key == room_id:
pagination_token = batch_token
else:
pagination_token = None
i = 0
# We keep looping and we keep filtering until we reach the limit
# or we run out of things.
# But only go around 5 times since otherwise synapse will be sad.
while len(room_events) < search_filter.limit() and i < 5:
i += 1
results = yield self.store.search_room(
room_id, search_term, keys, search_filter.limit() * 2,
pagination_token=pagination_token,
)
results_map = {r["event"].event_id: r for r in results}
rank_map.update({r["event"].event_id: r["rank"] for r in results})
filtered_events = search_filter.filter([
r["event"] for r in results
])
events = yield self._filter_events_for_client(
user.to_string(), filtered_events
)
room_events.extend(events)
room_events = room_events[:search_filter.limit()]
if len(results) < search_filter.limit() * 2:
pagination_token = None
break
else:
pagination_token = results[-1]["pagination_token"]
if room_events:
res = results_map[room_events[-1].event_id]
pagination_token = res["pagination_token"]
group = room_groups.setdefault(room_id, {})
if pagination_token:
next_batch = encode_base64("%s\n%s\n%s" % (
"room_id", room_id, pagination_token
))
group["next_batch"] = next_batch
if batch_token:
global_next_batch = next_batch
group["results"] = [e.event_id for e in room_events]
group["order"] = max(
e.origin_server_ts/1000 for e in room_events
if hasattr(e, "origin_server_ts")
)
allowed_events.extend(room_events)
# Normalize the group orders
if room_groups:
if len(room_groups) > 1:
mx = max(g["order"] for g in room_groups.values())
mn = min(g["order"] for g in room_groups.values())
for g in room_groups.values():
g["order"] = (g["order"] - mn) * 1.0 / (mx - mn)
else:
room_groups.values()[0]["order"] = 1
else:
# We should never get here due to the guard earlier.
raise NotImplementedError()
# If client has asked for "context" for each event (i.e. some surrounding
# events and state), fetch that
if event_context is not None:
now_token = yield self.hs.get_event_sources().get_current_token()
contexts = {}
for event in allowed_events:
res = yield self.store.get_events_around(
event.room_id, event.event_id, before_limit, after_limit
)
res["events_before"] = yield self._filter_events_for_client(
user.to_string(), res["events_before"]
)
res["events_after"] = yield self._filter_events_for_client(
user.to_string(), res["events_after"]
)
res["start"] = now_token.copy_and_replace(
"room_key", res["start"]
).to_string()
res["end"] = now_token.copy_and_replace(
"room_key", res["end"]
).to_string()
if include_profile:
senders = set(
ev.sender
for ev in itertools.chain(
res["events_before"], [event], res["events_after"]
)
)
if res["events_after"]:
last_event_id = res["events_after"][-1].event_id
else:
last_event_id = event.event_id
state = yield self.store.get_state_for_event(
last_event_id,
types=[(EventTypes.Member, sender) for sender in senders]
)
res["profile_info"] = {
s.state_key: {
"displayname": s.content.get("displayname", None),
"avatar_url": s.content.get("avatar_url", None),
}
for s in state.values()
if s.type == EventTypes.Member and s.state_key in senders
}
contexts[event.event_id] = res
else:
contexts = {}
# TODO: Add a limit
time_now = self.clock.time_msec()
for context in contexts.values():
context["events_before"] = [
serialize_event(e, time_now)
for e in context["events_before"]
]
context["events_after"] = [
serialize_event(e, time_now)
for e in context["events_after"]
]
state_results = {}
if include_state:
rooms = set(e.room_id for e in allowed_events)
for room_id in rooms:
state = yield self.state_handler.get_current_state(room_id)
state_results[room_id] = state.values()
state_results.values()
# We're now about to serialize the events. We should not make any
# blocking calls after this. Otherwise the 'age' will be wrong
results = {
e.event_id: {
"rank": rank_map[e.event_id],
"result": serialize_event(e, time_now),
"context": contexts.get(e.event_id, {}),
}
for e in allowed_events
}
logger.info("Found %d results", len(results))
rooms_cat_res = {
"results": results,
"count": len(results)
}
if state_results:
rooms_cat_res["state"] = {
room_id: [serialize_event(e, time_now) for e in state]
for room_id, state in state_results.items()
}
if room_groups and "room_id" in group_keys:
rooms_cat_res.setdefault("groups", {})["room_id"] = room_groups
if sender_group and "sender" in group_keys:
rooms_cat_res.setdefault("groups", {})["sender"] = sender_group
if global_next_batch:
rooms_cat_res["next_batch"] = global_next_batch
defer.returnValue({
"search_categories": {
"room_events": rooms_cat_res
}
})
def process_v2_response(
self, from_server, response_json, requested_ids=[],
):
"""Parse a 'Server Keys' structure from the result of a /key request
This is used to parse either the entirety of the response from
GET /_matrix/key/v2/server, or a single entry from the list returned by
POST /_matrix/key/v2/query.
Checks that each signature in the response that claims to come from the origin
server is valid. (Does not check that there actually is such a signature, for
some reason.)
Stores the json in server_keys_json so that it can be used for future responses
to /_matrix/key/v2/query.
Args:
from_server (str): the name of the server producing this result: either
the origin server for a /_matrix/key/v2/server request, or the notary
for a /_matrix/key/v2/query.
response_json (dict): the json-decoded Server Keys response object
requested_ids (iterable[str]): a list of the key IDs that were requested.
We will store the json for these key ids as well as any that are
actually in the response
Returns:
Deferred[dict[str, nacl.signing.VerifyKey]]:
map from key_id to key object
"""
time_now_ms = self.clock.time_msec()
response_keys = {}
verify_keys = {}
for key_id, key_data in response_json["verify_keys"].items():
if is_signing_algorithm_supported(key_id):
key_base64 = key_data["key"]
key_bytes = decode_base64(key_base64)
verify_key = decode_verify_key_bytes(key_id, key_bytes)
verify_key.time_added = time_now_ms
verify_keys[key_id] = verify_key
old_verify_keys = {}
for key_id, key_data in response_json["old_verify_keys"].items():
if is_signing_algorithm_supported(key_id):
key_base64 = key_data["key"]
key_bytes = decode_base64(key_base64)
verify_key = decode_verify_key_bytes(key_id, key_bytes)
verify_key.expired = key_data["expired_ts"]
verify_key.time_added = time_now_ms
old_verify_keys[key_id] = verify_key
server_name = response_json["server_name"]
for key_id in response_json["signatures"].get(server_name, {}):
if key_id not in response_json["verify_keys"]:
raise KeyLookupError(
"Key response must include verification keys for all"
" signatures"
)
if key_id in verify_keys:
verify_signed_json(
response_json,
server_name,
verify_keys[key_id]
)
signed_key_json = sign_json(
response_json,
self.config.server_name,
self.config.signing_key[0],
)
signed_key_json_bytes = encode_canonical_json(signed_key_json)
ts_valid_until_ms = signed_key_json[u"valid_until_ts"]
updated_key_ids = set(requested_ids)
updated_key_ids.update(verify_keys)
updated_key_ids.update(old_verify_keys)
response_keys.update(verify_keys)
response_keys.update(old_verify_keys)
yield logcontext.make_deferred_yieldable(defer.gatherResults(
[
run_in_background(
self.store.store_server_keys_json,
server_name=server_name,
key_id=key_id,
from_server=from_server,
ts_now_ms=time_now_ms,
ts_expires_ms=ts_valid_until_ms,
key_json_bytes=signed_key_json_bytes,
)
for key_id in updated_key_ids
],
consumeErrors=True,
).addErrback(unwrapFirstError))
defer.returnValue(response_keys)
from tests import unittest
from synapse.events.builder import EventBuilder
from synapse.crypto.event_signing import add_hashes_and_signatures
from unpaddedbase64 import decode_base64
import nacl.signing
# Perform these tests using given secret key so we get entirely deterministic
# signatures output that we can test against.
SIGNING_KEY_SEED = decode_base64(
"YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1"
)
KEY_ALG = "ed25519"
KEY_VER = 1
KEY_NAME = "%s:%d" % (KEY_ALG, KEY_VER)
HOSTNAME = "domain"
class EventSigningTestCase(unittest.TestCase):
def setUp(self):
self.signing_key = nacl.signing.SigningKey(SIGNING_KEY_SEED)
self.signing_key.alg = KEY_ALG
self.signing_key.version = KEY_VER
def from_token(cls, token):
return RoomListNextBatch(**{
cls.REVERSE_KEY_DICT[key]: val
for key, val in msgpack.loads(decode_base64(token)).items()
})
