def start_datastore(self, db_name): if not db_name: raise TypeError( "In order to start a datastore, you need to supply a db_name.") from nucypher.keystore import keystore from nucypher.keystore.db import Base from sqlalchemy.engine import create_engine engine = create_engine('sqlite:///{}'.format(db_name)) Base.metadata.create_all(engine) self.datastore = keystore.KeyStore(engine) self.db_engine = engine
def make_rest_app( db_filepath: str, this_node, serving_domains, log=Logger("http-application-layer") ) -> Tuple: forgetful_node_storage = ForgetfulNodeStorage(federated_only=this_node.federated_only) from nucypher.keystore import keystore from nucypher.keystore.db import Base from sqlalchemy.engine import create_engine log.info("Starting datastore {}".format(db_filepath)) # See: https://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#connect-strings if db_filepath: db_uri = f'sqlite:///{db_filepath}' else: db_uri = 'sqlite://' # TODO: Is this a sane default? See #667 engine = create_engine(db_uri) Base.metadata.create_all(engine) datastore = keystore.KeyStore(engine) db_engine = engine from nucypher.characters.lawful import Alice, Ursula _alice_class = Alice _node_class = Ursula rest_app = Flask("ursula-service") @rest_app.route("/public_information") def public_information(): """ REST endpoint for public keys and address. """ response = Response( response=bytes(this_node), mimetype='application/octet-stream') return response @rest_app.route('/node_metadata', methods=["GET"]) def all_known_nodes(): headers = {'Content-Type': 'application/octet-stream'} if this_node.known_nodes.checksum is NO_KNOWN_NODES: return Response(b"", headers=headers, status=204) known_nodes_bytestring = this_node.bytestring_of_known_nodes() signature = this_node.stamp(known_nodes_bytestring) return Response(bytes(signature) + known_nodes_bytestring, headers=headers) @rest_app.route('/node_metadata', methods=["POST"]) def node_metadata_exchange(): # If these nodes already have the same fleet state, no exchange is necessary. learner_fleet_state = request.args.get('fleet') if learner_fleet_state == this_node.known_nodes.checksum: log.debug("Learner already knew fleet state {}; doing nothing.".format(learner_fleet_state)) headers = {'Content-Type': 'application/octet-stream'} payload = this_node.known_nodes.snapshot() + bytes(FLEET_STATES_MATCH) signature = this_node.stamp(payload) return Response(bytes(signature) + payload, headers=headers) sprouts = _node_class.batch_from_bytes(request.data, registry=this_node.registry) # TODO: This logic is basically repeated in learn_from_teacher_node and remember_node. # Let's find a better way. #555 for node in sprouts: @crosstown_traffic() def learn_about_announced_nodes(): if node in this_node.known_nodes: if node.timestamp <= this_node.known_nodes[node.checksum_address].timestamp: return node.mature() try: node.verify_node(this_node.network_middleware.client, registry=this_node.registry, ) # Suspicion except node.SuspiciousActivity as e: # TODO: Include data about caller? # TODO: Account for possibility that stamp, rather than interface, was bad. # TODO: Maybe also record the bytes representation separately to disk? message = f"Suspicious Activity about {node}: {str(e)}. Announced via REST." log.warn(message) this_node.suspicious_activities_witnessed['vladimirs'].append(node) except NodeSeemsToBeDown as e: # This is a rather odd situation - this node *just* contacted us and asked to be verified. Where'd it go? Maybe a NAT problem? log.info(f"Node announced itself to us just now, but seems to be down: {node}. Response was {e}.") log.debug(f"Phantom node certificate: {node.certificate}") # Async Sentinel except Exception as e: log.critical(f"This exception really needs to be handled differently: {e}") raise # Believable else: log.info("Learned about previously unknown node: {}".format(node)) this_node.remember_node(node) # TODO: Record new fleet state # Cleanup finally: forgetful_node_storage.forget() # TODO: What's the right status code here? 202? Different if we already knew about the node? return all_known_nodes() @rest_app.route('/consider_arrangement', methods=['POST']) def consider_arrangement(): from nucypher.policy.policies import Arrangement arrangement = Arrangement.from_bytes(request.data) with ThreadedSession(db_engine) as session: new_policy_arrangement = datastore.add_policy_arrangement( arrangement.expiration.datetime(), id=arrangement.id.hex().encode(), alice_verifying_key=arrangement.alice.stamp, session=session, ) # TODO: Make the rest of this logic actually work - do something here # to decide if this Arrangement is worth accepting. headers = {'Content-Type': 'application/octet-stream'} # TODO: Make this a legit response #234. return Response(b"This will eventually be an actual acceptance of the arrangement.", headers=headers) @rest_app.route("/kFrag/<id_as_hex>", methods=['POST']) def set_policy(id_as_hex): """ REST endpoint for setting a kFrag. TODO: Instead of taking a Request, use the apistar typing system to type a payload and validate / split it. TODO: Validate that the kfrag being saved is pursuant to an approved Policy (see #121). """ policy_message_kit = UmbralMessageKit.from_bytes(request.data) alices_verifying_key = policy_message_kit.sender_verifying_key alice = _alice_class.from_public_keys(verifying_key=alices_verifying_key) try: cleartext = this_node.verify_from(alice, policy_message_kit, decrypt=True) except InvalidSignature: # TODO: Perhaps we log this? return Response(status_code=400) kfrag = KFrag.from_bytes(cleartext) if not kfrag.verify(signing_pubkey=alices_verifying_key): raise InvalidSignature("{} is invalid".format(kfrag)) with ThreadedSession(db_engine) as session: datastore.attach_kfrag_to_saved_arrangement( alice, id_as_hex, kfrag, session=session) # TODO: Sign the arrangement here. #495 return "" # TODO: Return A 200, with whatever policy metadata. @rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"]) def revoke_arrangement(id_as_hex): """ REST endpoint for revoking/deleting a KFrag from a node. """ from nucypher.policy.collections import Revocation revocation = Revocation.from_bytes(request.data) log.info("Received revocation: {} -- for arrangement {}".format(bytes(revocation).hex(), id_as_hex)) try: with ThreadedSession(db_engine) as session: # Verify the Notice was signed by Alice policy_arrangement = datastore.get_policy_arrangement( id_as_hex.encode(), session=session) alice_pubkey = UmbralPublicKey.from_bytes( policy_arrangement.alice_verifying_key.key_data) # Check that the request is the same for the provided revocation if id_as_hex != revocation.arrangement_id.hex(): log.debug("Couldn't identify an arrangement with id {}".format(id_as_hex)) return Response(status_code=400) elif revocation.verify_signature(alice_pubkey): datastore.del_policy_arrangement( id_as_hex.encode(), session=session) except (NotFound, InvalidSignature) as e: log.debug("Exception attempting to revoke: {}".format(e)) return Response(response='KFrag not found or revocation signature is invalid.', status=404) else: log.info("KFrag successfully removed.") return Response(response='KFrag deleted!', status=200) @rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"]) def reencrypt_via_rest(id_as_hex): # Get Policy Arrangement try: arrangement_id = binascii.unhexlify(id_as_hex) except (binascii.Error, TypeError): return Response(response=b'Invalid arrangement ID', status=405) try: with ThreadedSession(db_engine) as session: arrangement = datastore.get_policy_arrangement(arrangement_id=id_as_hex.encode(), session=session) except NotFound: return Response(response=arrangement_id, status=404) # Get KFrag kfrag = KFrag.from_bytes(arrangement.kfrag) # Get Work Order from nucypher.policy.collections import WorkOrder # Avoid circular import alice_verifying_key_bytes = arrangement.alice_verifying_key.key_data alice_verifying_key = UmbralPublicKey.from_bytes(alice_verifying_key_bytes) alice_address = canonical_address_from_umbral_key(alice_verifying_key) work_order_payload = request.data work_order = WorkOrder.from_rest_payload(arrangement_id=arrangement_id, rest_payload=work_order_payload, ursula=this_node, alice_address=alice_address) log.info(f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}") # Re-encrypt response = this_node._reencrypt(kfrag=kfrag, work_order=work_order, alice_verifying_key=alice_verifying_key) # Now, Ursula saves this workorder to her database... with ThreadedSession(db_engine): this_node.datastore.save_workorder(bob_verifying_key=bytes(work_order.bob.stamp), bob_signature=bytes(work_order.receipt_signature), arrangement_id=work_order.arrangement_id) headers = {'Content-Type': 'application/octet-stream'} return Response(headers=headers, response=response) @rest_app.route('/treasure_map/<treasure_map_id>') def provide_treasure_map(treasure_map_id): headers = {'Content-Type': 'application/octet-stream'} treasure_map_index = bytes.fromhex(treasure_map_id) try: treasure_map = this_node.treasure_maps[treasure_map_index] response = Response(bytes(treasure_map), headers=headers) log.info("{} providing TreasureMap {}".format(this_node.nickname, treasure_map_id)) except KeyError: log.info("{} doesn't have requested TreasureMap {}".format(this_node.stamp, treasure_map_id)) response = Response("No Treasure Map with ID {}".format(treasure_map_id), status=404, headers=headers) return response @rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST']) def receive_treasure_map(treasure_map_id): from nucypher.policy.collections import TreasureMap try: treasure_map = TreasureMap.from_bytes(bytes_representation=request.data, verify=True) except TreasureMap.InvalidSignature: do_store = False else: do_store = treasure_map.public_id() == treasure_map_id if do_store: log.info("{} storing TreasureMap {}".format(this_node, treasure_map_id)) # TODO 341 - what if we already have this TreasureMap? treasure_map_index = bytes.fromhex(treasure_map_id) this_node.treasure_maps[treasure_map_index] = treasure_map return Response(bytes(treasure_map), status=202) else: # TODO: Make this a proper 500 or whatever. log.info("Bad TreasureMap ID; not storing {}".format(treasure_map_id)) assert False @rest_app.route('/status/', methods=['GET']) def status(): if request.args.get('json'): payload = this_node.abridged_node_details() response = jsonify(payload) return response else: headers = {"Content-Type": "text/html", "charset": "utf-8"} previous_states = list(reversed(this_node.known_nodes.states.values()))[:5] # Mature every known node before rendering. for node in this_node.known_nodes: node.mature() try: content = status_template.render(this_node=this_node, known_nodes=this_node.known_nodes, previous_states=previous_states, domains=serving_domains, version=nucypher.__version__, checksum_address=this_node.checksum_address) except Exception as e: log.debug("Template Rendering Exception: ".format(str(e))) raise TemplateError(str(e)) from e return Response(response=content, headers=headers) return rest_app, datastore
def test_keystore(): engine = create_engine('sqlite:///:memory:') Base.metadata.create_all(engine) test_keystore = keystore.KeyStore(engine) yield test_keystore
def make_rest_app( db_filepath: str, network_middleware: RestMiddleware, federated_only: bool, treasure_map_tracker: dict, node_tracker: 'FleetStateTracker', node_bytes_caster: Callable, work_order_tracker: list, node_nickname: str, node_recorder: Callable, stamp: SignatureStamp, verifier: Callable, suspicious_activity_tracker: dict, serving_domains, log=Logger("http-application-layer")) -> Tuple: forgetful_node_storage = ForgetfulNodeStorage( federated_only=federated_only) from nucypher.keystore import keystore from nucypher.keystore.db import Base from sqlalchemy.engine import create_engine log.info("Starting datastore {}".format(db_filepath)) # See: https://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#connect-strings if db_filepath: db_uri = f'sqlite:///{db_filepath}' else: db_uri = 'sqlite://' # TODO: Is this a sane default? See #667 engine = create_engine(db_uri) Base.metadata.create_all(engine) datastore = keystore.KeyStore(engine) db_engine = engine from nucypher.characters.lawful import Alice, Ursula _alice_class = Alice _node_class = Ursula rest_app = Flask("ursula-service") @rest_app.route("/public_information") def public_information(): """ REST endpoint for public keys and address.. """ response = Response(response=node_bytes_caster(), mimetype='application/octet-stream') return response @rest_app.route('/node_metadata', methods=["GET"]) def all_known_nodes(): headers = {'Content-Type': 'application/octet-stream'} if node_tracker.checksum is NO_KNOWN_NODES: return Response(b"", headers=headers, status=204) payload = node_tracker.snapshot() ursulas_as_vbytes = (VariableLengthBytestring(n) for n in node_tracker) ursulas_as_bytes = bytes().join(bytes(u) for u in ursulas_as_vbytes) ursulas_as_bytes += VariableLengthBytestring(node_bytes_caster()) payload += ursulas_as_bytes signature = stamp(payload) return Response(bytes(signature) + payload, headers=headers) @rest_app.route('/node_metadata', methods=["POST"]) def node_metadata_exchange(): # If these nodes already have the same fleet state, no exchange is necessary. learner_fleet_state = request.args.get('fleet') if learner_fleet_state == node_tracker.checksum: log.debug( "Learner already knew fleet state {}; doing nothing.".format( learner_fleet_state)) headers = {'Content-Type': 'application/octet-stream'} payload = node_tracker.snapshot() + bytes(FLEET_STATES_MATCH) signature = stamp(payload) return Response(bytes(signature) + payload, headers=headers) nodes = _node_class.batch_from_bytes( request.data, federated_only=federated_only) # TODO: 466 # TODO: This logic is basically repeated in learn_from_teacher_node and remember_node. # Let's find a better way. #555 for node in nodes: if GLOBAL_DOMAIN not in serving_domains: if not serving_domains.intersection(node.serving_domains): continue # This node is not serving any of our domains. if node in node_tracker: if node.timestamp <= node_tracker[ node.checksum_public_address].timestamp: continue @crosstown_traffic() def learn_about_announced_nodes(): try: certificate_filepath = forgetful_node_storage.store_node_certificate( certificate=node.certificate) node.verify_node( network_middleware, accept_federated_only=federated_only, # TODO: 466 certificate_filepath=certificate_filepath) # Suspicion except node.SuspiciousActivity: # TODO: Include data about caller? # TODO: Account for possibility that stamp, rather than interface, was bad. # TODO: Maybe also record the bytes representation separately to disk? message = "Suspicious Activity: Discovered node with bad signature: {}. Announced via REST." log.warn(message) suspicious_activity_tracker['vladimirs'].append(node) # Async Sentinel except Exception as e: log.critical(str(e)) raise # Believable else: log.info( "Learned about previously unknown node: {}".format( node)) node_recorder(node) # TODO: Record new fleet state # Cleanup finally: forgetful_node_storage.forget() # TODO: What's the right status code here? 202? Different if we already knew about the node? return all_known_nodes() @rest_app.route('/consider_arrangement', methods=['POST']) def consider_arrangement(): from nucypher.policy.models import Arrangement arrangement = Arrangement.from_bytes(request.data) with ThreadedSession(db_engine) as session: new_policy_arrangement = datastore.add_policy_arrangement( arrangement.expiration.datetime(), id=arrangement.id.hex().encode(), alice_pubkey_sig=arrangement.alice.stamp, session=session, ) # TODO: Make the rest of this logic actually work - do something here # to decide if this Arrangement is worth accepting. headers = {'Content-Type': 'application/octet-stream'} # TODO: Make this a legit response #234. return Response( b"This will eventually be an actual acceptance of the arrangement.", headers=headers) @rest_app.route("/kFrag/<id_as_hex>", methods=['POST']) def set_policy(id_as_hex): """ REST endpoint for setting a kFrag. TODO: Instead of taking a Request, use the apistar typing system to type a payload and validate / split it. TODO: Validate that the kfrag being saved is pursuant to an approved Policy (see #121). """ policy_message_kit = UmbralMessageKit.from_bytes(request.data) alices_verifying_key = policy_message_kit.sender_pubkey_sig alice = _alice_class.from_public_keys( {SigningPower: alices_verifying_key}) try: cleartext = verifier(alice, policy_message_kit, decrypt=True) except InvalidSignature: # TODO: Perhaps we log this? return Response(status_code=400) kfrag = KFrag.from_bytes(cleartext) if not kfrag.verify(signing_pubkey=alices_verifying_key): raise InvalidSignature("{} is invalid".format(kfrag)) with ThreadedSession(db_engine) as session: datastore.attach_kfrag_to_saved_arrangement(alice, id_as_hex, kfrag, session=session) # TODO: Sign the arrangement here. #495 return "" # TODO: Return A 200, with whatever policy metadata. @rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"]) def revoke_arrangement(id_as_hex): """ REST endpoint for revoking/deleting a KFrag from a node. """ from nucypher.policy.models import Revocation revocation = Revocation.from_bytes(request.data) log.info("Received revocation: {} -- for arrangement {}".format( bytes(revocation).hex(), id_as_hex)) try: with ThreadedSession(db_engine) as session: # Verify the Notice was signed by Alice policy_arrangement = datastore.get_policy_arrangement( id_as_hex.encode(), session=session) alice_pubkey = UmbralPublicKey.from_bytes( policy_arrangement.alice_pubkey_sig.key_data) # Check that the request is the same for the provided revocation if id_as_hex != revocation.arrangement_id.hex(): log.debug( "Couldn't identify an arrangement with id {}".format( id_as_hex)) return Response(status_code=400) elif revocation.verify_signature(alice_pubkey): datastore.del_policy_arrangement(id_as_hex.encode(), session=session) except (NotFound, InvalidSignature) as e: log.debug("Exception attempting to revoke: {}".format(e)) return Response( response='KFrag not found or revocation signature is invalid.', status=404) else: log.info("KFrag successfully removed.") return Response(response='KFrag deleted!', status=200) @rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"]) def reencrypt_via_rest(id_as_hex): from nucypher.policy.models import WorkOrder # Avoid circular import arrangement_id = binascii.unhexlify(id_as_hex) work_order = WorkOrder.from_rest_payload(arrangement_id, request.data) log.info("Work Order from {}, signed {}".format( work_order.bob, work_order.receipt_signature)) with ThreadedSession(db_engine) as session: policy_arrangement = datastore.get_policy_arrangement( arrangement_id=id_as_hex.encode(), session=session) kfrag_bytes = policy_arrangement.kfrag # Careful! :-) verifying_key_bytes = policy_arrangement.alice_pubkey_sig.key_data # TODO: Push this to a lower level. Perhaps to Ursula character? #619 kfrag = KFrag.from_bytes(kfrag_bytes) alices_verifying_key = UmbralPublicKey.from_bytes(verifying_key_bytes) cfrag_byte_stream = b"" alices_address = canonical_address_from_umbral_key( alices_verifying_key) if not alices_address == work_order.alice_address: message = f"This Bob ({work_order.bob}) sent an Alice's ETH address " \ f"({work_order.alice_address}) that doesn't match " \ f"the one I have ({alices_address})." raise SuspiciousActivity(message) bob_pubkey = work_order.bob.stamp.as_umbral_pubkey() if not work_order.alice_address_signature.verify( message=alices_address, verifying_key=bob_pubkey): message = f"This Bob ({work_order.bob}) sent an invalid signature of Alice's ETH address" raise InvalidSignature(message) # This is Bob's signature of Alice's verifying key as ETH address. alice_address_signature = bytes(work_order.alice_address_signature) for capsule, capsule_signature in zip(work_order.capsules, work_order.capsule_signatures): # This is the capsule signed by Bob capsule_signature = bytes(capsule_signature) # Ursula signs on top of it. Now both are committed to the same capsule. # She signs Alice's address too. ursula_signature = stamp(capsule_signature + alice_address_signature) capsule.set_correctness_keys(verifying=alices_verifying_key) cfrag = pre.reencrypt(kfrag, capsule, metadata=bytes(ursula_signature)) log.info(f"Re-encrypting for {capsule}, made {cfrag}.") signature = stamp(bytes(cfrag) + bytes(capsule)) cfrag_byte_stream += VariableLengthBytestring(cfrag) + signature # TODO: Put this in Ursula's datastore work_order_tracker.append(work_order) headers = {'Content-Type': 'application/octet-stream'} return Response(response=cfrag_byte_stream, headers=headers) @rest_app.route('/treasure_map/<treasure_map_id>') def provide_treasure_map(treasure_map_id): headers = {'Content-Type': 'application/octet-stream'} treasure_map_bytes = keccak_digest(binascii.unhexlify(treasure_map_id)) try: treasure_map = treasure_map_tracker[treasure_map_bytes] response = Response(bytes(treasure_map), headers=headers) log.info("{} providing TreasureMap {}".format( node_nickname, treasure_map_id)) except KeyError: log.info("{} doesn't have requested TreasureMap {}".format( stamp, treasure_map_id)) response = Response( "No Treasure Map with ID {}".format(treasure_map_id), status=404, headers=headers) return response @rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST']) def receive_treasure_map(treasure_map_id): from nucypher.policy.models import TreasureMap try: treasure_map = TreasureMap.from_bytes( bytes_representation=request.data, verify=True) except TreasureMap.InvalidSignature: do_store = False else: do_store = treasure_map.public_id() == treasure_map_id if do_store: log.info("{} storing TreasureMap {}".format( stamp, treasure_map_id)) # # # # # TODO: Now that the DHT is retired, let's do this another way. # self.dht_server.set_now(binascii.unhexlify(treasure_map_id), # constants.BYTESTRING_IS_TREASURE_MAP + bytes(treasure_map)) # # # # # TODO 341 - what if we already have this TreasureMap? treasure_map_tracker[keccak_digest( binascii.unhexlify(treasure_map_id))] = treasure_map return Response(bytes(treasure_map), status=202) else: # TODO: Make this a proper 500 or whatever. log.info( "Bad TreasureMap ID; not storing {}".format(treasure_map_id)) assert False @rest_app.route('/status') def status(): # TODO: Seems very strange to deserialize *this node* when we can just pass it in. # Might be a sign that we need to rethnk this composition. headers = {"Content-Type": "text/html", "charset": "utf-8"} this_node = _node_class.from_bytes(node_bytes_caster(), federated_only=federated_only) previous_states = list(reversed(node_tracker.states.values()))[:5] try: content = status_template.render(this_node=this_node, known_nodes=node_tracker, previous_states=previous_states) except Exception as e: log.debug("Template Rendering Exception: ".format(str(e))) raise TemplateError(str(e)) from e return Response(response=content, headers=headers) return rest_app, datastore
def __init__( self, db_name, db_filepath, network_middleware, federated_only, treasure_map_tracker, node_tracker, node_bytes_caster, work_order_tracker, node_recorder, stamp, verifier, suspicious_activity_tracker, certificate_dir, ) -> None: self.network_middleware = network_middleware self.federated_only = federated_only self._treasure_map_tracker = treasure_map_tracker self._work_order_tracker = work_order_tracker self._node_tracker = node_tracker self._node_bytes_caster = node_bytes_caster self._node_recorder = node_recorder self._stamp = stamp self._verifier = verifier self._suspicious_activity_tracker = suspicious_activity_tracker self._certificate_dir = certificate_dir self.datastore = None routes = [ Route('/kFrag/{id_as_hex}', 'POST', self.set_policy), Route('/kFrag/{id_as_hex}/reencrypt', 'POST', self.reencrypt_via_rest), Route('/public_information', 'GET', self.public_information), Route('/node_metadata', 'GET', self.all_known_nodes), Route('/node_metadata', 'POST', self.node_metadata_exchange), Route('/consider_arrangement', 'POST', self.consider_arrangement), Route('/treasure_map/{treasure_map_id}', 'GET', self.provide_treasure_map), Route('/status', 'GET', self.status), Route('/treasure_map/{treasure_map_id}', 'POST', self.receive_treasure_map), ] self.rest_app = App(routes=routes) self.db_name = db_name self.db_filepath = db_filepath from nucypher.keystore import keystore from nucypher.keystore.db import Base from sqlalchemy.engine import create_engine self.log.info("Starting datastore {}".format(self.db_filepath)) engine = create_engine('sqlite:///{}'.format(self.db_filepath)) Base.metadata.create_all(engine) self.datastore = keystore.KeyStore(engine) self.db_engine = engine from nucypher.characters.lawful import Alice, Ursula self._alice_class = Alice self._node_class = Ursula with open(os.path.join(TEMPLATES_DIR, "basic_status.j2"), "r") as f: _status_template_content = f.read() self._status_template = Template(_status_template_content)