def response_json_object(self): verify_keys = {} for key in self.config.signing_key: verify_key_bytes = key.verify_key.encode() key_id = "%s:%s" % (key.alg, key.version) verify_keys[key_id] = { u"key": encode_base64(verify_key_bytes) } old_verify_keys = {} for key_id, key in self.config.old_signing_keys.items(): verify_key_bytes = key.encode() old_verify_keys[key_id] = { u"key": encode_base64(verify_key_bytes), u"expired_ts": key.expired_ts, } tls_fingerprints = self.config.tls_fingerprints json_object = { u"valid_until_ts": self.valid_until_ts, u"server_name": self.config.server_name, u"verify_keys": verify_keys, u"old_verify_keys": old_verify_keys, u"tls_fingerprints": tls_fingerprints, } for key in self.config.signing_key: json_object = sign_json( json_object, self.config.server_name, key, ) return json_object
def sign_json(json_object, signature_name, signing_key): """Sign the JSON object. Stores the signature in json_object["signatures"]. Args: json_object (dict): The JSON object to sign. signature_name (str): The name of the signing entity. signing_key (syutil.crypto.SigningKey): The key to sign the JSON with. Returns: The modified, signed JSON object.""" signatures = json_object.pop("signatures", {}) unsigned = json_object.pop("unsigned", None) message_bytes = encode_canonical_json(json_object) signed = signing_key.sign(message_bytes) signature_base64 = encode_base64(signed.signature) key_id = "%s:%s" % (signing_key.alg, signing_key.version) signatures.setdefault(signature_name, {})[key_id] = signature_base64 # logger.debug("SIGNING: %s %s %s", signature_name, key_id, message_bytes) json_object["signatures"] = signatures if unsigned is not None: json_object["unsigned"] = unsigned return json_object
def read_certificate_from_disk(self, require_cert_and_key): """ Read the certificates and private key from disk. Args: require_cert_and_key (bool): set to True to throw an error if the certificate and key file are not given """ if require_cert_and_key: self.tls_private_key = self.read_tls_private_key() self.tls_certificate = self.read_tls_certificate() elif self.tls_certificate_file: # we only need the certificate for the tls_fingerprints. Reload it if we # can, but it's not a fatal error if we can't. try: self.tls_certificate = self.read_tls_certificate() except Exception as e: logger.info( "Unable to read TLS certificate (%s). Ignoring as no " "tls listeners enabled.", e, ) self.tls_fingerprints = list(self._original_tls_fingerprints) if self.tls_certificate: # Check that our own certificate is included in the list of fingerprints # and include it if it is not. x509_certificate_bytes = crypto.dump_certificate( crypto.FILETYPE_ASN1, self.tls_certificate ) sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest()) sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints) if sha256_fingerprint not in sha256_fingerprints: self.tls_fingerprints.append({u"sha256": sha256_fingerprint})
def select_pdus(cursor): cursor.execute( "SELECT pdu_id, origin FROM pdus ORDER BY depth ASC" ) ids = cursor.fetchall() pdu_tuples = store._get_pdu_tuples(cursor, ids) pdus = [Pdu.from_pdu_tuple(p) for p in pdu_tuples] reference_hashes = {} for pdu in pdus: try: if pdu.prev_pdus: print "PROCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus for pdu_id, origin, hashes in pdu.prev_pdus: ref_alg, ref_hsh = reference_hashes[(pdu_id, origin)] hashes[ref_alg] = encode_base64(ref_hsh) store._store_prev_pdu_hash_txn(cursor, pdu.pdu_id, pdu.origin, pdu_id, origin, ref_alg, ref_hsh) print "SUCCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus pdu = add_event_pdu_content_hash(pdu) ref_alg, ref_hsh = compute_pdu_event_reference_hash(pdu) reference_hashes[(pdu.pdu_id, pdu.origin)] = (ref_alg, ref_hsh) store._store_pdu_reference_hash_txn(cursor, pdu.pdu_id, pdu.origin, ref_alg, ref_hsh) for alg, hsh_base64 in pdu.hashes.items(): print alg, hsh_base64 store._store_pdu_content_hash_txn(cursor, pdu.pdu_id, pdu.origin, alg, decode_base64(hsh_base64)) except: print "FAILED_", pdu.pdu_id, pdu.origin, pdu.prev_pdus
def check_event_content_hash(event, hash_algorithm=hashlib.sha256): """Check whether the hash for this PDU matches the contents""" name, expected_hash = compute_content_hash(event.get_pdu_json(), hash_algorithm) logger.debug("Expecting hash: %s", encode_base64(expected_hash)) # some malformed events lack a 'hashes'. Protect against it being missing # or a weird type by basically treating it the same as an unhashed event. hashes = event.get("hashes") if not isinstance(hashes, dict): raise SynapseError(400, "Malformed 'hashes'", Codes.UNAUTHORIZED) if name not in hashes: raise SynapseError( 400, "Algorithm %s not in hashes %s" % ( name, list(hashes), ), Codes.UNAUTHORIZED, ) message_hash_base64 = hashes[name] try: message_hash_bytes = decode_base64(message_hash_base64) except Exception: raise SynapseError( 400, "Invalid base64: %s" % (message_hash_base64,), Codes.UNAUTHORIZED, ) return message_hash_bytes == expected_hash
def add_event_hashes(self, event_ids): hashes = yield self.get_event_reference_hashes(event_ids) hashes = { e_id: {k: encode_base64(v) for k, v in h.items() if k == "sha256"} for e_id, h in hashes.items() } defer.returnValue(list(hashes.items()))
def encode_signing_key_base64(key): """Encode a signing key as base64 Args: key (SigningKey): A signing key to encode. Returns: base64 encoded string. """ return encode_base64(key.encode())
def encode_verify_key_base64(key): """Encode a verify key as base64 Args: key (VerifyKey): A signing key to encode. Returns: base64 encoded string. """ return encode_base64(key.encode())
def get_server_verify_key_v2_direct(self, server_name, key_ids): keys = {} for requested_key_id in key_ids: if requested_key_id in keys: continue (response, tls_certificate) = yield fetch_server_key( server_name, self.hs.tls_server_context_factory, path=(b"/_matrix/key/v2/server/%s" % ( urllib.quote(requested_key_id), )).encode("ascii"), ) if (u"signatures" not in response or server_name not in response[u"signatures"]): raise KeyLookupError("Key response not signed by remote server") if "tls_fingerprints" not in response: raise KeyLookupError("Key response missing TLS fingerprints") certificate_bytes = crypto.dump_certificate( crypto.FILETYPE_ASN1, tls_certificate ) sha256_fingerprint = hashlib.sha256(certificate_bytes).digest() sha256_fingerprint_b64 = encode_base64(sha256_fingerprint) response_sha256_fingerprints = set() for fingerprint in response[u"tls_fingerprints"]: if u"sha256" in fingerprint: response_sha256_fingerprints.add(fingerprint[u"sha256"]) if sha256_fingerprint_b64 not in response_sha256_fingerprints: raise KeyLookupError("TLS certificate not allowed by fingerprints") response_keys = yield self.process_v2_response( from_server=server_name, requested_ids=[requested_key_id], response_json=response, ) keys.update(response_keys) yield logcontext.make_deferred_yieldable(defer.gatherResults( [ run_in_background( self.store_keys, server_name=key_server_name, from_server=server_name, verify_keys=verify_keys, ) for key_server_name, verify_keys in keys.items() ], consumeErrors=True ).addErrback(unwrapFirstError)) defer.returnValue(keys)
def event_id(self): # We have to import this here as otherwise we get an import loop which # is hard to break. from synapse.crypto.event_signing import compute_event_reference_hash if self._event_id: return self._event_id self._event_id = "$" + encode_base64(compute_event_reference_hash(self)[1]) return self._event_id
def render_GET(self, request): err, args = get_args(request, ("public_key",)) if err: return json.dumps(err) pubKey = self.sydent.keyring.ed25519.verify_key pubKeyBase64 = encode_base64(pubKey.encode()) return json.dumps({'valid': args["public_key"] == pubKeyBase64})
def select_v1_keys(connection): cursor = connection.cursor() cursor.execute("SELECT server_name, key_id, verify_key FROM server_signature_keys") rows = cursor.fetchall() cursor.close() results = {} for server_name, key_id, verify_key in rows: results.setdefault(server_name, {})[key_id] = encode_base64(verify_key) return results
def test_sign_and_verify(self): self.assertIn('signatures', self.signed) self.assertIn('Alice', self.signed['signatures']) self.assertIn('mock:test', self.signed['signatures']['Alice']) self.assertEqual( self.signed['signatures']['Alice']['mock:test'], encode_base64(b'x_______') ) self.assertEqual(self.sigkey.signed_bytes, b'{"foo":"bar"}') verify_signed_json(self.signed, 'Alice', self.verkey)
def response_json_object(self): verify_keys = {} for key in self.config.signing_key: verify_key_bytes = key.verify_key.encode() key_id = "%s:%s" % (key.alg, key.version) verify_keys[key_id] = { u"key": encode_base64(verify_key_bytes) } old_verify_keys = {} for key in self.config.old_signing_keys: key_id = "%s:%s" % (key.alg, key.version) verify_key_bytes = key.encode() old_verify_keys[key_id] = { u"key": encode_base64(verify_key_bytes), u"expired_ts": key.expired, } x509_certificate_bytes = crypto.dump_certificate( crypto.FILETYPE_ASN1, self.config.tls_certificate ) sha256_fingerprint = sha256(x509_certificate_bytes).digest() json_object = { u"valid_until_ts": self.valid_until_ts, u"server_name": self.config.server_name, u"verify_keys": verify_keys, u"old_verify_keys": old_verify_keys, u"tls_fingerprints": [{ u"sha256": encode_base64(sha256_fingerprint), }] } for key in self.config.signing_key: json_object = sign_json( json_object, self.config.server_name, key, ) return json_object
def add_event_hashes(self, event_ids): hashes = yield self.get_event_reference_hashes( event_ids ) hashes = [ { k: encode_base64(v) for k, v in h.items() if k == "sha256" } for h in hashes ] defer.returnValue(zip(event_ids, hashes))
def response_json_object(server_config): verify_keys = {} for key in server_config.signing_key: verify_key_bytes = key.verify_key.encode() key_id = "%s:%s" % (key.alg, key.version) verify_keys[key_id] = encode_base64(verify_key_bytes) x509_certificate_bytes = crypto.dump_certificate( crypto.FILETYPE_ASN1, server_config.tls_certificate ) json_object = { u"server_name": server_config.server_name, u"verify_keys": verify_keys, u"tls_certificate": encode_base64(x509_certificate_bytes) } for key in server_config.signing_key: json_object = sign_json( json_object, server_config.server_name, key, ) return json_object
def response_json_object(self): verify_keys = {} for key in self.config.signing_key: verify_key_bytes = key.verify_key.encode() key_id = "%s:%s" % (key.alg, key.version) verify_keys[key_id] = { u"key": encode_base64(verify_key_bytes) } old_verify_keys = {} for key_id, key in self.config.old_signing_keys.items(): verify_key_bytes = key.encode() old_verify_keys[key_id] = { u"key": encode_base64(verify_key_bytes), u"expired_ts": key.expired_ts, } if self.config.no_tls: tls_fingerprints = [] else: tls_fingerprints = self.config.tls_fingerprints json_object = { u"valid_until_ts": self.valid_until_ts, u"server_name": self.config.server_name, u"verify_keys": verify_keys, u"old_verify_keys": old_verify_keys, u"tls_fingerprints": tls_fingerprints, } for key in self.config.signing_key: json_object = sign_json( json_object, self.config.server_name, key, ) return json_object
def _get_latest_event_ids_and_hashes_in_room(self, txn, room_id): sql = ( "SELECT e.event_id, e.depth FROM events as e " "INNER JOIN event_forward_extremities as f " "ON e.event_id = f.event_id " "AND e.room_id = f.room_id " "WHERE f.room_id = ?" ) txn.execute(sql, (room_id,)) results = [] for event_id, depth in txn.fetchall(): hashes = self._get_event_reference_hashes_txn(txn, event_id) prev_hashes = { k: encode_base64(v) for k, v in hashes.items() if k == "sha256" } results.append((event_id, prev_hashes, depth)) return results
def main(): parser = argparse.ArgumentParser() parser.add_argument( "input_json", nargs="?", type=argparse.FileType('r'), default=sys.stdin ) args = parser.parse_args() logging.basicConfig() event_json = dictobj(json.load(args.input_json)) algorithms = {"sha256": hashlib.sha256} for alg_name in event_json.hashes: if check_event_content_hash(event_json, algorithms[alg_name]): print("PASS content hash %s" % (alg_name,)) else: print("FAIL content hash %s" % (alg_name,)) for algorithm in algorithms.values(): name, h_bytes = compute_event_reference_hash(event_json, algorithm) print("Reference hash %s: %s" % (name, encode_base64(h_bytes)))
def check_event_content_hash(event, hash_algorithm=hashlib.sha256): """Check whether the hash for this PDU matches the contents""" name, expected_hash = compute_content_hash(event, hash_algorithm) logger.debug("Expecting hash: %s", encode_base64(expected_hash)) if name not in event.hashes: raise SynapseError( 400, "Algorithm %s not in hashes %s" % ( name, list(event.hashes), ), Codes.UNAUTHORIZED, ) message_hash_base64 = event.hashes[name] try: message_hash_bytes = decode_base64(message_hash_base64) except: raise SynapseError( 400, "Invalid base64: %s" % (message_hash_base64,), Codes.UNAUTHORIZED, ) return message_hash_bytes == expected_hash
def add_hashes_and_signatures(event_dict, signature_name, signing_key, hash_algorithm=hashlib.sha256): """Add content hash and sign the event Args: event_dict (dict): The event to add hashes to and sign signature_name (str): The name of the entity signing the event (typically the server's hostname). signing_key (syutil.crypto.SigningKey): The key to sign with hash_algorithm: A hasher from `hashlib`, e.g. hashlib.sha256, to use to hash the event """ name, digest = compute_content_hash(event_dict, hash_algorithm=hash_algorithm) event_dict.setdefault("hashes", {})[name] = encode_base64(digest) event_dict["signatures"] = compute_event_signature( event_dict, signature_name=signature_name, signing_key=signing_key, )
def read_config(self, config): self.tls_certificate = self.read_tls_certificate( config.get("tls_certificate_path") ) self.tls_certificate_file = config.get("tls_certificate_path") self.no_tls = config.get("no_tls", False) if self.no_tls: self.tls_private_key = None else: self.tls_private_key = self.read_tls_private_key( config.get("tls_private_key_path") ) self.tls_dh_params_path = self.check_file( config.get("tls_dh_params_path"), "tls_dh_params" ) self.tls_fingerprints = config["tls_fingerprints"] # Check that our own certificate is included in the list of fingerprints # and include it if it is not. x509_certificate_bytes = crypto.dump_certificate( crypto.FILETYPE_ASN1, self.tls_certificate ) sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest()) sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints) if sha256_fingerprint not in sha256_fingerprints: self.tls_fingerprints.append({u"sha256": sha256_fingerprint}) # This config option applies to non-federation HTTP clients # (e.g. for talking to recaptcha, identity servers, and such) # It should never be used in production, and is intended for # use only when running tests. self.use_insecure_ssl_client_just_for_testing_do_not_use = config.get( "use_insecure_ssl_client_just_for_testing_do_not_use" )
def add_hashes_and_signatures(event, signature_name, signing_key, hash_algorithm=hashlib.sha256): # if hasattr(event, "old_state_events"): # state_json_bytes = encode_canonical_json( # [e.event_id for e in event.old_state_events.values()] # ) # hashed = hash_algorithm(state_json_bytes) # event.state_hash = { # hashed.name: encode_base64(hashed.digest()) # } name, digest = compute_content_hash(event, hash_algorithm=hash_algorithm) if not hasattr(event, "hashes"): event.hashes = {} event.hashes[name] = encode_base64(digest) event.signatures = compute_event_signature( event, signature_name=signature_name, signing_key=signing_key, )
def render_GET(self, request): pubKey = self.sydent.keyring.ed25519.verify_key pubKeyBase64 = encode_base64(pubKey.encode()) return json.dumps({'public_key': pubKeyBase64})
def render_POST(self, request): send_cors(request) err = require_args(request, ("medium", "address", "room_id", "sender",)) if err: return json.dumps(err) medium = request.args["medium"][0] address = request.args["address"][0] roomId = request.args["room_id"][0] sender = request.args["sender"][0] globalAssocStore = GlobalAssociationStore(self.sydent) mxid = globalAssocStore.getMxid(medium, address) if mxid: request.setResponseCode(400) return json.dumps({ "errcode": "THREEPID_IN_USE", "error": "Binding already known", "mxid": mxid, }) if medium != "email": request.setResponseCode(400) return json.dumps({ "errcode": "M_UNRECOGNIZED", "error": "Didn't understand medium '%s'" % (medium,), }) token = self._randomString(128) tokenStore = JoinTokenStore(self.sydent) ephemeralPrivateKey = nacl.signing.SigningKey.generate() ephemeralPublicKey = ephemeralPrivateKey.verify_key ephemeralPrivateKeyBase64 = encode_base64(ephemeralPrivateKey.encode(), True) ephemeralPublicKeyBase64 = encode_base64(ephemeralPublicKey.encode(), True) tokenStore.storeEphemeralPublicKey(ephemeralPublicKeyBase64) tokenStore.storeToken(medium, address, roomId, sender, token) substitutions = {} for key, values in request.args.items(): if len(values) == 1 and type(values[0]) == str: substitutions[key] = values[0] substitutions["token"] = token required = [ 'sender_display_name', 'token', 'room_name', 'bracketed_room_name', 'room_avatar_url', 'sender_display_name', 'guest_user_id', 'guest_access_token', ] for k in required: substitutions.setdefault(k, '') substitutions["ephemeral_private_key"] = ephemeralPrivateKeyBase64 if substitutions["room_name"] != '': substitutions["bracketed_room_name"] = "(%s)" % substitutions["room_name"] subject_header = Header(self.sydent.cfg.get('email', 'email.invite.subject', raw=True) % substitutions, 'utf8') substitutions["subject_header_value"] = subject_header.encode() sendEmail(self.sydent, "email.invite_template", address, substitutions) pubKey = self.sydent.keyring.ed25519.verify_key pubKeyBase64 = encode_base64(pubKey.encode()) baseUrl = "%s/_matrix/identity/api/v1" % (self.sydent.cfg.get('http', 'client_http_base'),) keysToReturn = [] keysToReturn.append({ "public_key": pubKeyBase64, "key_validity_url": baseUrl + "/pubkey/isvalid", }) keysToReturn.append({ "public_key": ephemeralPublicKeyBase64, "key_validity_url": baseUrl + "/pubkey/ephemeral/isvalid", }) resp = { "token": token, "public_key": pubKeyBase64, "public_keys": keysToReturn, "display_name": self.redact(address), } return json.dumps(resp)
def test_verify_fail(self): self.signed['signatures']['Alice']['mock:test'] = encode_base64( b'not a signature' ) with self.assertRaises(SignatureVerifyException): verify_signed_json(self.signed, 'Alice', self.verkey)
def to_token(self): return encode_base64(msgpack.dumps({ self.KEY_DICT[key]: val for key, val in self._asdict().items() }))
def search(self, user, content, batch=None): """Performs a full text search for a user. Args: user (UserID) content (dict): Search parameters batch (str): The next_batch parameter. Used for pagination. Returns: dict to be returned to the client with results of search """ if not self.hs.config.enable_search: raise SynapseError(400, "Search is disabled on this homeserver") batch_group = None batch_group_key = None batch_token = None if batch: try: b = decode_base64(batch).decode('ascii') batch_group, batch_group_key, batch_token = b.split("\n") assert batch_group is not None assert batch_group_key is not None assert batch_token is not None except Exception: raise SynapseError(400, "Invalid batch") logger.info( "Search batch properties: %r, %r, %r", batch_group, batch_group_key, batch_token, ) logger.info("Search content: %s", content) try: room_cat = content["search_categories"]["room_events"] # The actual thing to query in FTS search_term = room_cat["search_term"] # Which "keys" to search over in FTS query keys = room_cat.get("keys", [ "content.body", "content.name", "content.topic", ]) # Filter to apply to results filter_dict = room_cat.get("filter", {}) # What to order results by (impacts whether pagination can be doen) order_by = room_cat.get("order_by", "rank") # Return the current state of the rooms? include_state = room_cat.get("include_state", False) # Include context around each event? event_context = room_cat.get( "event_context", None ) # Group results together? May allow clients to paginate within a # group group_by = room_cat.get("groupings", {}).get("group_by", {}) group_keys = [g["key"] for g in group_by] if event_context is not None: before_limit = int(event_context.get( "before_limit", 5 )) after_limit = int(event_context.get( "after_limit", 5 )) # Return the historic display name and avatar for the senders # of the events? include_profile = bool(event_context.get("include_profile", False)) except KeyError: raise SynapseError(400, "Invalid search query") if order_by not in ("rank", "recent"): raise SynapseError(400, "Invalid order by: %r" % (order_by,)) if set(group_keys) - {"room_id", "sender"}: raise SynapseError( 400, "Invalid group by keys: %r" % (set(group_keys) - {"room_id", "sender"},) ) search_filter = Filter(filter_dict) # TODO: Search through left rooms too rooms = yield self.store.get_rooms_for_user_where_membership_is( user.to_string(), membership_list=[Membership.JOIN], # membership_list=[Membership.JOIN, Membership.LEAVE, Membership.Ban], ) room_ids = set(r.room_id for r in rooms) # If doing a subset of all rooms seearch, check if any of the rooms # are from an upgraded room, and search their contents as well if search_filter.rooms: historical_room_ids = [] for room_id in search_filter.rooms: # Add any previous rooms to the search if they exist ids = yield self.get_old_rooms_from_upgraded_room(room_id) historical_room_ids += ids # Prevent any historical events from being filtered search_filter = search_filter.with_room_ids(historical_room_ids) room_ids = search_filter.filter_rooms(room_ids) if batch_group == "room_id": room_ids.intersection_update({batch_group_key}) if not room_ids: defer.returnValue({ "search_categories": { "room_events": { "results": [], "count": 0, "highlights": [], } } }) rank_map = {} # event_id -> rank of event allowed_events = [] room_groups = {} # Holds result of grouping by room, if applicable sender_group = {} # Holds result of grouping by sender, if applicable # Holds the next_batch for the entire result set if one of those exists global_next_batch = None highlights = set() count = None if order_by == "rank": search_result = yield self.store.search_msgs( room_ids, search_term, keys ) count = search_result["count"] if search_result["highlights"]: highlights.update(search_result["highlights"]) results = search_result["results"] results_map = {r["event"].event_id: r for r in results} rank_map.update({r["event"].event_id: r["rank"] for r in results}) filtered_events = search_filter.filter([r["event"] for r in results]) events = yield filter_events_for_client( self.store, user.to_string(), filtered_events ) events.sort(key=lambda e: -rank_map[e.event_id]) allowed_events = events[:search_filter.limit()] for e in allowed_events: rm = room_groups.setdefault(e.room_id, { "results": [], "order": rank_map[e.event_id], }) rm["results"].append(e.event_id) s = sender_group.setdefault(e.sender, { "results": [], "order": rank_map[e.event_id], }) s["results"].append(e.event_id) elif order_by == "recent": room_events = [] i = 0 pagination_token = batch_token # We keep looping and we keep filtering until we reach the limit # or we run out of things. # But only go around 5 times since otherwise synapse will be sad. while len(room_events) < search_filter.limit() and i < 5: i += 1 search_result = yield self.store.search_rooms( room_ids, search_term, keys, search_filter.limit() * 2, pagination_token=pagination_token, ) if search_result["highlights"]: highlights.update(search_result["highlights"]) count = search_result["count"] results = search_result["results"] results_map = {r["event"].event_id: r for r in results} rank_map.update({r["event"].event_id: r["rank"] for r in results}) filtered_events = search_filter.filter([ r["event"] for r in results ]) events = yield filter_events_for_client( self.store, user.to_string(), filtered_events ) room_events.extend(events) room_events = room_events[:search_filter.limit()] if len(results) < search_filter.limit() * 2: pagination_token = None break else: pagination_token = results[-1]["pagination_token"] for event in room_events: group = room_groups.setdefault(event.room_id, { "results": [], }) group["results"].append(event.event_id) if room_events and len(room_events) >= search_filter.limit(): last_event_id = room_events[-1].event_id pagination_token = results_map[last_event_id]["pagination_token"] # We want to respect the given batch group and group keys so # that if people blindly use the top level `next_batch` token # it returns more from the same group (if applicable) rather # than reverting to searching all results again. if batch_group and batch_group_key: global_next_batch = encode_base64(("%s\n%s\n%s" % ( batch_group, batch_group_key, pagination_token )).encode('ascii')) else: global_next_batch = encode_base64(("%s\n%s\n%s" % ( "all", "", pagination_token )).encode('ascii')) for room_id, group in room_groups.items(): group["next_batch"] = encode_base64(("%s\n%s\n%s" % ( "room_id", room_id, pagination_token )).encode('ascii')) allowed_events.extend(room_events) else: # We should never get here due to the guard earlier. raise NotImplementedError() logger.info("Found %d events to return", len(allowed_events)) # If client has asked for "context" for each event (i.e. some surrounding # events and state), fetch that if event_context is not None: now_token = yield self.hs.get_event_sources().get_current_token() contexts = {} for event in allowed_events: res = yield self.store.get_events_around( event.room_id, event.event_id, before_limit, after_limit, ) logger.info( "Context for search returned %d and %d events", len(res["events_before"]), len(res["events_after"]), ) res["events_before"] = yield filter_events_for_client( self.store, user.to_string(), res["events_before"] ) res["events_after"] = yield filter_events_for_client( self.store, user.to_string(), res["events_after"] ) res["start"] = now_token.copy_and_replace( "room_key", res["start"] ).to_string() res["end"] = now_token.copy_and_replace( "room_key", res["end"] ).to_string() if include_profile: senders = set( ev.sender for ev in itertools.chain( res["events_before"], [event], res["events_after"] ) ) if res["events_after"]: last_event_id = res["events_after"][-1].event_id else: last_event_id = event.event_id state_filter = StateFilter.from_types( [(EventTypes.Member, sender) for sender in senders] ) state = yield self.store.get_state_for_event( last_event_id, state_filter ) res["profile_info"] = { s.state_key: { "displayname": s.content.get("displayname", None), "avatar_url": s.content.get("avatar_url", None), } for s in state.values() if s.type == EventTypes.Member and s.state_key in senders } contexts[event.event_id] = res else: contexts = {} # TODO: Add a limit time_now = self.clock.time_msec() for context in contexts.values(): context["events_before"] = ( yield self._event_serializer.serialize_events( context["events_before"], time_now, ) ) context["events_after"] = ( yield self._event_serializer.serialize_events( context["events_after"], time_now, ) ) state_results = {} if include_state: rooms = set(e.room_id for e in allowed_events) for room_id in rooms: state = yield self.state_handler.get_current_state(room_id) state_results[room_id] = list(state.values()) state_results.values() # We're now about to serialize the events. We should not make any # blocking calls after this. Otherwise the 'age' will be wrong results = [] for e in allowed_events: results.append({ "rank": rank_map[e.event_id], "result": (yield self._event_serializer.serialize_event(e, time_now)), "context": contexts.get(e.event_id, {}), }) rooms_cat_res = { "results": results, "count": count, "highlights": list(highlights), } if state_results: s = {} for room_id, state in state_results.items(): s[room_id] = yield self._event_serializer.serialize_events( state, time_now, ) rooms_cat_res["state"] = s if room_groups and "room_id" in group_keys: rooms_cat_res.setdefault("groups", {})["room_id"] = room_groups if sender_group and "sender" in group_keys: rooms_cat_res.setdefault("groups", {})["sender"] = sender_group if global_next_batch: rooms_cat_res["next_batch"] = global_next_batch defer.returnValue({ "search_categories": { "room_events": rooms_cat_res } })
def search(self, user, content, batch=None): """Performs a full text search for a user. Args: user (UserID) content (dict): Search parameters batch (str): The next_batch parameter. Used for pagination. Returns: dict to be returned to the client with results of search """ batch_group = None batch_group_key = None batch_token = None if batch: try: b = decode_base64(batch) batch_group, batch_group_key, batch_token = b.split("\n") assert batch_group is not None assert batch_group_key is not None assert batch_token is not None except: raise SynapseError(400, "Invalid batch") try: room_cat = content["search_categories"]["room_events"] # The actual thing to query in FTS search_term = room_cat["search_term"] # Which "keys" to search over in FTS query keys = room_cat.get("keys", [ "content.body", "content.name", "content.topic", ]) # Filter to apply to results filter_dict = room_cat.get("filter", {}) # What to order results by (impacts whether pagination can be doen) order_by = room_cat.get("order_by", "rank") # Return the current state of the rooms? include_state = room_cat.get("include_state", False) # Include context around each event? event_context = room_cat.get( "event_context", None ) # Group results together? May allow clients to paginate within a # group group_by = room_cat.get("groupings", {}).get("group_by", {}) group_keys = [g["key"] for g in group_by] if event_context is not None: before_limit = int(event_context.get( "before_limit", 5 )) after_limit = int(event_context.get( "after_limit", 5 )) # Return the historic display name and avatar for the senders # of the events? include_profile = bool(event_context.get("include_profile", False)) except KeyError: raise SynapseError(400, "Invalid search query") if order_by not in ("rank", "recent"): raise SynapseError(400, "Invalid order by: %r" % (order_by,)) if set(group_keys) - {"room_id", "sender"}: raise SynapseError( 400, "Invalid group by keys: %r" % (set(group_keys) - {"room_id", "sender"},) ) search_filter = Filter(filter_dict) # TODO: Search through left rooms too rooms = yield self.store.get_rooms_for_user_where_membership_is( user.to_string(), membership_list=[Membership.JOIN], # membership_list=[Membership.JOIN, Membership.LEAVE, Membership.Ban], ) room_ids = set(r.room_id for r in rooms) room_ids = search_filter.filter_rooms(room_ids) if batch_group == "room_id": room_ids.intersection_update({batch_group_key}) rank_map = {} # event_id -> rank of event allowed_events = [] room_groups = {} # Holds result of grouping by room, if applicable sender_group = {} # Holds result of grouping by sender, if applicable # Holds the next_batch for the entire result set if one of those exists global_next_batch = None if order_by == "rank": results = yield self.store.search_msgs( room_ids, search_term, keys ) results_map = {r["event"].event_id: r for r in results} rank_map.update({r["event"].event_id: r["rank"] for r in results}) filtered_events = search_filter.filter([r["event"] for r in results]) events = yield self._filter_events_for_client( user.to_string(), filtered_events ) events.sort(key=lambda e: -rank_map[e.event_id]) allowed_events = events[:search_filter.limit()] for e in allowed_events: rm = room_groups.setdefault(e.room_id, { "results": [], "order": rank_map[e.event_id], }) rm["results"].append(e.event_id) s = sender_group.setdefault(e.sender, { "results": [], "order": rank_map[e.event_id], }) s["results"].append(e.event_id) elif order_by == "recent": # In this case we specifically loop through each room as the given # limit applies to each room, rather than a global list. # This is not necessarilly a good idea. for room_id in room_ids: room_events = [] if batch_group == "room_id" and batch_group_key == room_id: pagination_token = batch_token else: pagination_token = None i = 0 # We keep looping and we keep filtering until we reach the limit # or we run out of things. # But only go around 5 times since otherwise synapse will be sad. while len(room_events) < search_filter.limit() and i < 5: i += 1 results = yield self.store.search_room( room_id, search_term, keys, search_filter.limit() * 2, pagination_token=pagination_token, ) results_map = {r["event"].event_id: r for r in results} rank_map.update({r["event"].event_id: r["rank"] for r in results}) filtered_events = search_filter.filter([ r["event"] for r in results ]) events = yield self._filter_events_for_client( user.to_string(), filtered_events ) room_events.extend(events) room_events = room_events[:search_filter.limit()] if len(results) < search_filter.limit() * 2: pagination_token = None break else: pagination_token = results[-1]["pagination_token"] if room_events: res = results_map[room_events[-1].event_id] pagination_token = res["pagination_token"] group = room_groups.setdefault(room_id, {}) if pagination_token: next_batch = encode_base64("%s\n%s\n%s" % ( "room_id", room_id, pagination_token )) group["next_batch"] = next_batch if batch_token: global_next_batch = next_batch group["results"] = [e.event_id for e in room_events] group["order"] = max( e.origin_server_ts/1000 for e in room_events if hasattr(e, "origin_server_ts") ) allowed_events.extend(room_events) # Normalize the group orders if room_groups: if len(room_groups) > 1: mx = max(g["order"] for g in room_groups.values()) mn = min(g["order"] for g in room_groups.values()) for g in room_groups.values(): g["order"] = (g["order"] - mn) * 1.0 / (mx - mn) else: room_groups.values()[0]["order"] = 1 else: # We should never get here due to the guard earlier. raise NotImplementedError() # If client has asked for "context" for each event (i.e. some surrounding # events and state), fetch that if event_context is not None: now_token = yield self.hs.get_event_sources().get_current_token() contexts = {} for event in allowed_events: res = yield self.store.get_events_around( event.room_id, event.event_id, before_limit, after_limit ) res["events_before"] = yield self._filter_events_for_client( user.to_string(), res["events_before"] ) res["events_after"] = yield self._filter_events_for_client( user.to_string(), res["events_after"] ) res["start"] = now_token.copy_and_replace( "room_key", res["start"] ).to_string() res["end"] = now_token.copy_and_replace( "room_key", res["end"] ).to_string() if include_profile: senders = set( ev.sender for ev in itertools.chain( res["events_before"], [event], res["events_after"] ) ) if res["events_after"]: last_event_id = res["events_after"][-1].event_id else: last_event_id = event.event_id state = yield self.store.get_state_for_event( last_event_id, types=[(EventTypes.Member, sender) for sender in senders] ) res["profile_info"] = { s.state_key: { "displayname": s.content.get("displayname", None), "avatar_url": s.content.get("avatar_url", None), } for s in state.values() if s.type == EventTypes.Member and s.state_key in senders } contexts[event.event_id] = res else: contexts = {} # TODO: Add a limit time_now = self.clock.time_msec() for context in contexts.values(): context["events_before"] = [ serialize_event(e, time_now) for e in context["events_before"] ] context["events_after"] = [ serialize_event(e, time_now) for e in context["events_after"] ] state_results = {} if include_state: rooms = set(e.room_id for e in allowed_events) for room_id in rooms: state = yield self.state_handler.get_current_state(room_id) state_results[room_id] = state.values() state_results.values() # We're now about to serialize the events. We should not make any # blocking calls after this. Otherwise the 'age' will be wrong results = { e.event_id: { "rank": rank_map[e.event_id], "result": serialize_event(e, time_now), "context": contexts.get(e.event_id, {}), } for e in allowed_events } logger.info("Found %d results", len(results)) rooms_cat_res = { "results": results, "count": len(results) } if state_results: rooms_cat_res["state"] = { room_id: [serialize_event(e, time_now) for e in state] for room_id, state in state_results.items() } if room_groups and "room_id" in group_keys: rooms_cat_res.setdefault("groups", {})["room_id"] = room_groups if sender_group and "sender" in group_keys: rooms_cat_res.setdefault("groups", {})["sender"] = sender_group if global_next_batch: rooms_cat_res["next_batch"] = global_next_batch defer.returnValue({ "search_categories": { "room_events": rooms_cat_res } })
def fingerprint(certificate): finger = hashlib.sha256(certificate) return {"sha256": encode_base64(finger.digest())}
def get_server_verify_key_v1_direct(self, server_name, key_ids): """Finds a verification key for the server with one of the key ids. Args: server_name (str): The name of the server to fetch a key for. keys_ids (list of str): The key_ids to check for. """ # Try to fetch the key from the remote server. (response, tls_certificate) = yield fetch_server_key( server_name, self.hs.tls_server_context_factory ) # Check the response. x509_certificate_bytes = crypto.dump_certificate( crypto.FILETYPE_ASN1, tls_certificate ) if ("signatures" not in response or server_name not in response["signatures"]): raise ValueError("Key response not signed by remote server") if "tls_certificate" not in response: raise ValueError("Key response missing TLS certificate") tls_certificate_b64 = response["tls_certificate"] if encode_base64(x509_certificate_bytes) != tls_certificate_b64: raise ValueError("TLS certificate doesn't match") # Cache the result in the datastore. time_now_ms = self.clock.time_msec() verify_keys = {} for key_id, key_base64 in response["verify_keys"].items(): if is_signing_algorithm_supported(key_id): key_bytes = decode_base64(key_base64) verify_key = decode_verify_key_bytes(key_id, key_bytes) verify_key.time_added = time_now_ms verify_keys[key_id] = verify_key for key_id in response["signatures"][server_name]: if key_id not in response["verify_keys"]: raise ValueError( "Key response must include verification keys for all" " signatures" ) if key_id in verify_keys: verify_signed_json( response, server_name, verify_keys[key_id] ) yield self.store.store_server_certificate( server_name, server_name, time_now_ms, tls_certificate, ) yield self.store_keys( server_name=server_name, from_server=server_name, verify_keys=verify_keys, ) defer.returnValue(verify_keys)