def mock_or_real_datastore(request): if request.param: yield datastore.Datastore(MOCK_DB) else: temp_path = tempfile.mkdtemp() yield datastore.Datastore(temp_path) shutil.rmtree(temp_path)
def test_workorder_model(): temp_path = tempfile.mkdtemp() storage = datastore.Datastore(temp_path) bob_keypair = keypairs.SigningKeypair(generate_keys_if_needed=True) arrangement_id_hex = 'beef' bob_verifying_key = bob_keypair.pubkey bob_signature = bob_keypair.sign(b'test') # Test create with storage.describe(Workorder, arrangement_id_hex, writeable=True) as work_order: work_order.arrangement_id = bytes.fromhex(arrangement_id_hex) work_order.bob_verifying_key = bob_verifying_key work_order.bob_signature = bob_signature with storage.describe(Workorder, arrangement_id_hex) as work_order: assert work_order.arrangement_id == bytes.fromhex(arrangement_id_hex) assert work_order.bob_verifying_key == bob_verifying_key assert work_order.bob_signature == bob_signature # Test delete with storage.describe(Workorder, arrangement_id_hex, writeable=True) as work_order: work_order.delete() # Should be deleted now. with pytest.raises(AttributeError): should_error = work_order.arrangement_id
def test_policy_arrangement_model(): temp_path = tempfile.mkdtemp() storage = datastore.Datastore(temp_path) arrangement_id_hex = 'beef' expiration = maya.now() alice_verifying_key = keypairs.SigningKeypair( generate_keys_if_needed=True).pubkey # TODO: Leaving out KFrag for now since I don't have an easy way to grab one. with storage.describe(PolicyArrangement, arrangement_id_hex, writeable=True) as policy_arrangement: policy_arrangement.arrangement_id = bytes.fromhex(arrangement_id_hex) policy_arrangement.expiration = expiration policy_arrangement.alice_verifying_key = alice_verifying_key with storage.describe(PolicyArrangement, arrangement_id_hex) as policy_arrangement: assert policy_arrangement.arrangement_id == bytes.fromhex( arrangement_id_hex) assert policy_arrangement.expiration == expiration assert policy_arrangement.alice_verifying_key == alice_verifying_key # Now let's `delete` it with storage.describe(PolicyArrangement, arrangement_id_hex, writeable=True) as policy_arrangement: policy_arrangement.delete() # Should be deleted now. with pytest.raises(AttributeError): should_error = policy_arrangement.arrangement_id
def test_treasure_map_model(): temp_path = tempfile.mkdtemp() storage = datastore.Datastore(temp_path) hrac = 'beef' fake_treasure_map_data = b'My Little TreasureMap' with storage.describe(TreasureMap, hrac, writeable=True) as treasure_map: treasure_map.treasure_map = fake_treasure_map_data with storage.describe(TreasureMap, hrac) as treasure_map: assert treasure_map.treasure_map == b'My Little TreasureMap' # Test delete with storage.describe(TreasureMap, hrac, writeable=True) as treasure_map: treasure_map.delete() # Should be deleted now. with pytest.raises(AttributeError): should_error = treasure_map.treasure_map
def test_datastore(): test_datastore = datastore.Datastore(tempfile.mkdtemp()) yield test_datastore
def test_datastore_create(): temp_path = tempfile.mkdtemp() storage = datastore.Datastore(temp_path) assert storage.LMDB_MAP_SIZE == 1_000_000_000_000 assert storage.db_path == temp_path assert storage._Datastore__db_env.path() == temp_path
def make_rest_app( db_filepath: str, this_node, serving_domains, log=Logger("http-application-layer") ) -> Tuple: forgetful_node_storage = ForgetfulNodeStorage(federated_only=this_node.federated_only) from nucypher.datastore import datastore from nucypher.datastore.db import Base from sqlalchemy.engine import create_engine log.info("Starting datastore {}".format(db_filepath)) # See: https://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#connect-strings if db_filepath: db_uri = f'sqlite:///{db_filepath}' else: db_uri = 'sqlite://' # TODO: Is this a sane default? See #667 engine = create_engine(db_uri) Base.metadata.create_all(engine) datastore = datastore.Datastore(engine) db_engine = engine from nucypher.characters.lawful import Alice, Ursula _alice_class = Alice _node_class = Ursula rest_app = Flask("ursula-service") @rest_app.route("/public_information") def public_information(): """ REST endpoint for public keys and address. """ response = Response( response=bytes(this_node), mimetype='application/octet-stream') return response @rest_app.route('/node_metadata', methods=["GET"]) def all_known_nodes(): headers = {'Content-Type': 'application/octet-stream'} if this_node.known_nodes.checksum is NO_KNOWN_NODES: return Response(b"", headers=headers, status=204) known_nodes_bytestring = this_node.bytestring_of_known_nodes() signature = this_node.stamp(known_nodes_bytestring) return Response(bytes(signature) + known_nodes_bytestring, headers=headers) @rest_app.route('/node_metadata', methods=["POST"]) def node_metadata_exchange(): # If these nodes already have the same fleet state, no exchange is necessary. learner_fleet_state = request.args.get('fleet') if learner_fleet_state == this_node.known_nodes.checksum: log.debug("Learner already knew fleet state {}; doing nothing.".format(learner_fleet_state)) headers = {'Content-Type': 'application/octet-stream'} payload = this_node.known_nodes.snapshot() + bytes(FLEET_STATES_MATCH) signature = this_node.stamp(payload) return Response(bytes(signature) + payload, headers=headers) sprouts = _node_class.batch_from_bytes(request.data, registry=this_node.registry) # TODO: This logic is basically repeated in learn_from_teacher_node and remember_node. # Let's find a better way. #555 for node in sprouts: @crosstown_traffic() def learn_about_announced_nodes(): if node in this_node.known_nodes: if node.timestamp <= this_node.known_nodes[node.checksum_address].timestamp: return node.mature() try: node.verify_node(this_node.network_middleware.client, registry=this_node.registry, ) # Suspicion except node.SuspiciousActivity as e: # 355 # TODO: Include data about caller? # TODO: Account for possibility that stamp, rather than interface, was bad. # TODO: Maybe also record the bytes representation separately to disk? message = f"Suspicious Activity about {node}: {str(e)}. Announced via REST." log.warn(message) this_node.suspicious_activities_witnessed['vladimirs'].append(node) except NodeSeemsToBeDown as e: # This is a rather odd situation - this node *just* contacted us and asked to be verified. Where'd it go? Maybe a NAT problem? log.info(f"Node announced itself to us just now, but seems to be down: {node}. Response was {e}.") log.debug(f"Phantom node certificate: {node.certificate}") # Async Sentinel except Exception as e: log.critical(f"This exception really needs to be handled differently: {e}") raise # Believable else: log.info("Learned about previously unknown node: {}".format(node)) this_node.remember_node(node) # TODO: Record new fleet state # Cleanup finally: forgetful_node_storage.forget() # TODO: What's the right status code here? 202? Different if we already knew about the node? return all_known_nodes() @rest_app.route('/consider_arrangement', methods=['POST']) def consider_arrangement(): from nucypher.policy.policies import Arrangement arrangement = Arrangement.from_bytes(request.data) # TODO: Look at the expiration and figure out if we're even staking that long. 1701 with ThreadedSession(db_engine) as session: new_policy_arrangement = datastore.add_policy_arrangement( arrangement.expiration.datetime(), arrangement_id=arrangement.id.hex().encode(), alice_verifying_key=arrangement.alice.stamp, session=session, ) # TODO: Fine, we'll add the arrangement here, but if we never hear from Alice again to enact it, # we need to prune it at some point. #1700 headers = {'Content-Type': 'application/octet-stream'} # TODO: Make this a legit response #234. return Response(b"This will eventually be an actual acceptance of the arrangement.", headers=headers) @rest_app.route("/kFrag/<id_as_hex>", methods=['POST']) def set_policy(id_as_hex): """ REST endpoint for setting a kFrag. """ policy_message_kit = UmbralMessageKit.from_bytes(request.data) alices_verifying_key = policy_message_kit.sender_verifying_key alice = _alice_class.from_public_keys(verifying_key=alices_verifying_key) try: cleartext = this_node.verify_from(alice, policy_message_kit, decrypt=True) except InvalidSignature: # TODO: Perhaps we log this? Essentially 355. return Response(status_code=400) if not this_node.federated_only: # This splitter probably belongs somewhere canonical. transaction_splitter = BytestringSplitter(32) tx, kfrag_bytes = transaction_splitter(cleartext, return_remainder=True) try: # Get all of the arrangements and verify that we'll be paid. # TODO: We'd love for this part to be impossible to reduce the risk of collusion. #1274 arranged_addresses = this_node.policy_agent.fetch_arrangement_addresses_from_policy_txid(tx, timeout=this_node.synchronous_query_timeout) except TimeExhausted: # Alice didn't pay. Return response with that weird status code. this_node.suspicious_activities_witnessed['freeriders'].append((alice, f"No transaction matching {tx}.")) return Response(status=402) this_node_has_been_arranged = this_node.checksum_address in arranged_addresses if not this_node_has_been_arranged: this_node.suspicious_activities_witnessed['freeriders'].append((alice, f"The transaction {tx} does not list me as a Worker - it lists {arranged_addresses}.")) return Response(status=402) else: _tx = NO_BLOCKCHAIN_CONNECTION kfrag_bytes = cleartext kfrag = KFrag.from_bytes(kfrag_bytes) if not kfrag.verify(signing_pubkey=alices_verifying_key): raise InvalidSignature("{} is invalid".format(kfrag)) with ThreadedSession(db_engine) as session: datastore.attach_kfrag_to_saved_arrangement( alice, id_as_hex, kfrag, session=session) # TODO: Sign the arrangement here. #495 return "" # TODO: Return A 200, with whatever policy metadata. @rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"]) def revoke_arrangement(id_as_hex): """ REST endpoint for revoking/deleting a KFrag from a node. """ from nucypher.policy.collections import Revocation revocation = Revocation.from_bytes(request.data) log.info("Received revocation: {} -- for arrangement {}".format(bytes(revocation).hex(), id_as_hex)) try: with ThreadedSession(db_engine) as session: # Verify the Notice was signed by Alice policy_arrangement = datastore.get_policy_arrangement( id_as_hex.encode(), session=session) alice_pubkey = UmbralPublicKey.from_bytes( policy_arrangement.alice_verifying_key.key_data) # Check that the request is the same for the provided revocation if id_as_hex != revocation.arrangement_id.hex(): log.debug("Couldn't identify an arrangement with id {}".format(id_as_hex)) return Response(status_code=400) elif revocation.verify_signature(alice_pubkey): datastore.del_policy_arrangement( id_as_hex.encode(), session=session) except (NotFound, InvalidSignature) as e: log.debug("Exception attempting to revoke: {}".format(e)) return Response(response='KFrag not found or revocation signature is invalid.', status=404) else: log.info("KFrag successfully removed.") return Response(response='KFrag deleted!', status=200) @rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"]) def reencrypt_via_rest(id_as_hex): # Get Policy Arrangement try: arrangement_id = binascii.unhexlify(id_as_hex) except (binascii.Error, TypeError): return Response(response=b'Invalid arrangement ID', status=405) try: with ThreadedSession(db_engine) as session: arrangement = datastore.get_policy_arrangement(arrangement_id=id_as_hex.encode(), session=session) except NotFound: return Response(response=arrangement_id, status=404) # Get KFrag # TODO: Yeah, well, what if this arrangement hasn't been enacted? 1702 kfrag = KFrag.from_bytes(arrangement.kfrag) # Get Work Order from nucypher.policy.collections import WorkOrder # Avoid circular import alice_verifying_key_bytes = arrangement.alice_verifying_key.key_data alice_verifying_key = UmbralPublicKey.from_bytes(alice_verifying_key_bytes) alice_address = canonical_address_from_umbral_key(alice_verifying_key) work_order_payload = request.data work_order = WorkOrder.from_rest_payload(arrangement_id=arrangement_id, rest_payload=work_order_payload, ursula=this_node, alice_address=alice_address) log.info(f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}") # Re-encrypt response = this_node._reencrypt(kfrag=kfrag, work_order=work_order, alice_verifying_key=alice_verifying_key) # Now, Ursula saves this workorder to her database... with ThreadedSession(db_engine): this_node.datastore.save_workorder(bob_verifying_key=bytes(work_order.bob.stamp), bob_signature=bytes(work_order.receipt_signature), arrangement_id=work_order.arrangement_id) headers = {'Content-Type': 'application/octet-stream'} return Response(headers=headers, response=response) @rest_app.route('/treasure_map/<treasure_map_id>') def provide_treasure_map(treasure_map_id): headers = {'Content-Type': 'application/octet-stream'} treasure_map_index = bytes.fromhex(treasure_map_id) try: treasure_map = this_node.treasure_maps[treasure_map_index] response = Response(bytes(treasure_map), headers=headers) log.info("{} providing TreasureMap {}".format(this_node.nickname, treasure_map_id)) except KeyError: log.info("{} doesn't have requested TreasureMap {}".format(this_node.stamp, treasure_map_id)) response = Response("No Treasure Map with ID {}".format(treasure_map_id), status=404, headers=headers) return response @rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST']) def receive_treasure_map(treasure_map_id): from nucypher.policy.collections import TreasureMap try: treasure_map = TreasureMap.from_bytes(bytes_representation=request.data, verify=True) except TreasureMap.InvalidSignature: do_store = False else: # TODO: If we include the policy ID in this check, does that prevent map spam? 1736 do_store = treasure_map.public_id() == treasure_map_id if do_store: log.info("{} storing TreasureMap {}".format(this_node, treasure_map_id)) # TODO 341 - what if we already have this TreasureMap? treasure_map_index = bytes.fromhex(treasure_map_id) this_node.treasure_maps[treasure_map_index] = treasure_map return Response(bytes(treasure_map), status=202) else: # TODO: Make this a proper 500 or whatever. #341 log.info("Bad TreasureMap ID; not storing {}".format(treasure_map_id)) assert False @rest_app.route('/status/', methods=['GET']) def status(): if request.args.get('json'): payload = this_node.abridged_node_details() response = jsonify(payload) return response else: headers = {"Content-Type": "text/html", "charset": "utf-8"} previous_states = list(reversed(this_node.known_nodes.states.values()))[:5] # Mature every known node before rendering. for node in this_node.known_nodes: node.mature() try: content = status_template.render(this_node=this_node, known_nodes=this_node.known_nodes, previous_states=previous_states, domains=serving_domains, version=nucypher.__version__, checksum_address=this_node.checksum_address) except Exception as e: log.debug("Template Rendering Exception: ".format(str(e))) raise TemplateError(str(e)) from e return Response(response=content, headers=headers) return rest_app, datastore
def test_datastore(): engine = create_engine('sqlite:///:memory:') Base.metadata.create_all(engine) test_datastore = datastore.Datastore(engine) yield test_datastore
def test_datastore_describe(): temp_path = tempfile.mkdtemp() storage = datastore.Datastore(temp_path) assert storage.LMDB_MAP_SIZE == 1_000_000_000_000 assert storage.db_path == temp_path assert storage._Datastore__db_env.path() == temp_path # # Tests for `Datastore.describe` # # Getting writeable access to a record can be done by setting `writeable` to `True`. # `writeable` is, by default, `False`. # In the event a record doesn't exist, this will raise a `RecordNotFound` error iff `writeable=False`. with pytest.raises(datastore.RecordNotFound): with storage.describe(TestRecord, 'test_id') as test_record: should_error = test_record.test # Reading a non-existent field from a writeable record is an error with pytest.raises(datastore.DatastoreTransactionError): with storage.describe(TestRecord, 'test_id', writeable=True) as test_record: what_is_this = test_record.test # Writing to a, previously nonexistent record, with a valid field works! with storage.describe(TestRecord, 'test_id', writeable=True) as test_record: test_record.test = b'test data' assert test_record.test == b'test data' # Check that you can't reuse the record instance to write outside the context manager with pytest.raises(TypeError): test_record.test = b'should not write' # Nor can you read outside the context manager with pytest.raises(lmdb.Error): should_error = test_record.test # Records can also have ints as IDs with storage.describe(TestRecord, 1337, writeable=True) as test_record: test_record.test = b'test int ID' assert test_record.test == b'test int ID' # Writing to a non-existent field errors with pytest.raises(datastore.DatastoreTransactionError): with storage.describe(TestRecord, 'test_id', writeable=True) as test_record: test_record.nonexistent_field = b'this will error' # Writing the wrong type to a field errors with pytest.raises(datastore.DatastoreTransactionError): with storage.describe(TestRecord, 'test_id', writeable=True) as test_record: test_record.test = 1234 # Check that nothing was written with storage.describe(TestRecord, 'test_id') as test_record: assert test_record.test != 1234 # Any unhandled errors in the context manager results in a transaction abort with pytest.raises(datastore.DatastoreTransactionError): with storage.describe(TestRecord, 'test_id', writeable=True) as test_record: # Valid write test_record.test = b'this will not persist' # Erroneous write causing an abort test_record.nonexistent = b'causes an error and aborts the write' # Check that nothing was written from the aborted transaction above. with storage.describe(TestRecord, 'test_id') as test_record: assert test_record.test == b'test data' # However, a handled error will not cause an abort. with storage.describe(TestRecord, 'test_id', writeable=True) as test_record: # Valid operation test_record.test = b'this will persist' try: # Maybe we don't know that this field exists or not? # Erroneous, but handled, operation -- doesn't cause an abort should_error = test_record.bad_read except TypeError: pass # Because we handled the `TypeError`, the write persists. with storage.describe(TestRecord, 'test_id') as test_record: assert test_record.test == b'this will persist' # Be aware: if you don't handle _all the errors_, then the transaction will abort: with pytest.raises(datastore.DatastoreTransactionError): with storage.describe(TestRecord, 'test_id', writeable=True) as test_record: # Valid operation test_record.test = b'this will not persist' try: # We handle this one correctly # Erroneous, but handled, operation -- doesn't cause an abort this_will_not_abort = test_record.bad_read except TypeError: pass # However, we don't handle this one correctly # Erroneous UNHANDLED operation -- causes an abort this_WILL_abort = test_record.bad_read # The valid operation did not persist due to the unhandled error, despite # the other one being handled. with storage.describe(TestRecord, 'test_id') as test_record: assert test_record.test != b'this will not persist' # An applicable demonstration: # Let's imagine we don't know if a `TestRecord` identified by `new_id` exists. # If we want to conditionally modify it, we can do as follows: with storage.describe(TestRecord, 'new_id', writeable=True) as new_test_record: try: # Assume the record exists my_test_data = new_test_record.test # Do something with my_test_data except AttributeError: # We handle the case that there's no record for `new_test_record.test` # and write to it. new_test_record.test = b'now it exists :)' # And proof that it worked: with storage.describe(TestRecord, 'new_id') as new_test_record: assert new_test_record.test == b'now it exists :)'
def test_datastore_query_by(): temp_path = tempfile.mkdtemp() storage = datastore.Datastore(temp_path) # Make two test record classes class FooRecord(DatastoreRecord): _foo = RecordField(bytes) class BarRecord(DatastoreRecord): _foo = RecordField(bytes) _bar = RecordField(bytes) # We won't add this one class NoRecord(DatastoreRecord): _nothing = RecordField(bytes) # Create them with storage.describe(FooRecord, 1, writeable=True) as rec: rec.foo = b'one record' with storage.describe(FooRecord, 'two', writeable=True) as rec: rec.foo = b'another record' with storage.describe(FooRecord, 'three', writeable=True) as rec: rec.foo = b'another record' with storage.describe(BarRecord, 1, writeable=True) as rec: rec.bar = b'one record' with storage.describe(BarRecord, 'two', writeable=True) as rec: rec.foo = b'foo two record' rec.bar = b'two record' # Let's query! with storage.query_by(FooRecord) as records: assert len(records) == 3 assert type(records) == list assert records[0]._DatastoreRecord__writeable is False assert records[1]._DatastoreRecord__writeable is False assert records[2]._DatastoreRecord__writeable is False # Try with BarRecord with storage.query_by(BarRecord) as records: assert len(records) == 2 # Try to query for non-existent records with pytest.raises(datastore.RecordNotFound): with storage.query_by(NoRecord) as records: assert len(records) == 'this never gets executed cause it raises' # Queries without writeable are read only with pytest.raises(datastore.DatastoreTransactionError): with storage.query_by(FooRecord) as records: records[0].foo = b'this should error' # Let's query by specific record and field with storage.query_by(BarRecord, filter_field='foo') as records: assert len(records) == 1 # Query for a non-existent field in an existing record with pytest.raises(datastore.RecordNotFound): with storage.query_by(FooRecord, filter_field='bar') as records: assert len(records) == 'this never gets executed cause it raises' # Query for a non-existent field that is _similar to an existing field_ with pytest.raises(datastore.RecordNotFound): with storage.query_by(FooRecord, filter_field='fo') as records: assert len(records) == 'this never gets executed cause it raises' # Query for a field with a filtering function # When querying with a field _and_ a filtering function, the `filter_func` # callable is given the field value you specified. # We throw a `isinstance` in there to ensure that the type given is a field value and not a record filter_func = lambda field_val: not isinstance( field_val, DatastoreRecord) and field_val == b'another record' with storage.query_by(FooRecord, filter_field='foo', filter_func=filter_func) as records: assert len(records) == 2 assert records[0].foo == b'another record' assert records[1].foo == b'another record' # Query with _only_ a filter func. # This filter_func will receive a `DatastoreRecord` instance that is readonly filter_func = lambda field_rec: isinstance( field_rec, DatastoreRecord) and field_rec.foo == b'one record' with storage.query_by(FooRecord, filter_func=filter_func) as records: assert len(records) == 1 assert records[0].foo == b'one record' # This record isn't writeable with pytest.raises(TypeError): records[0].foo = b'this will error' # Make a writeable query on BarRecord with storage.query_by(BarRecord, writeable=True) as records: records[0].bar = b'this writes' records[1].bar = b'this writes' assert records[0].bar == b'this writes' assert records[1].bar == b'this writes' # Writeable queries on non-existant records error with pytest.raises(datastore.RecordNotFound): with storage.query_by(NoRecord, writeable=True) as records: assert len(records) == 'this never gets executed'
def make_rest_app( db_filepath: str, this_node, serving_domains, log=Logger("http-application-layer")) -> Tuple: forgetful_node_storage = ForgetfulNodeStorage( federated_only=this_node.federated_only) from nucypher.datastore import datastore log.info("Starting datastore {}".format(db_filepath)) datastore = datastore.Datastore(db_filepath) from nucypher.characters.lawful import Alice, Ursula _alice_class = Alice _node_class = Ursula rest_app = Flask("ursula-service") rest_app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD_CONTENT_LENGTH @rest_app.route("/public_information") def public_information(): """ REST endpoint for public keys and address. """ response = Response(response=bytes(this_node), mimetype='application/octet-stream') return response @rest_app.route("/ping", methods=['POST']) def ping(): """ Asks this node: "Can you access my public information endpoint"? """ try: requesting_ursula = Ursula.from_bytes(request.data, registry=this_node.registry) requesting_ursula.mature() except ValueError: # (ValueError) return Response({'error': 'Invalid Ursula'}, status=400) else: initiator_address, initiator_port = tuple( requesting_ursula.rest_interface) # Compare requester and posted Ursula information request_address = request.environ['REMOTE_ADDR'] if request_address != initiator_address: return Response({'error': 'Suspicious origin address'}, status=400) # # Make a Sandwich # try: # Fetch and store initiator's teacher certificate. certificate = this_node.network_middleware.get_certificate( host=initiator_address, port=initiator_port) certificate_filepath = this_node.node_storage.store_node_certificate( certificate=certificate) requesting_ursula_bytes = this_node.network_middleware.client.node_information( host=initiator_address, port=initiator_port, certificate_filepath=certificate_filepath) except NodeSeemsToBeDown: return Response({'error': 'Unreachable node'}, status=400) # ... toasted # Compare the results of the outer POST with the inner GET... yum if requesting_ursula_bytes == request.data: return Response(status=200) else: return Response({'error': 'Suspicious node'}, status=400) @rest_app.route('/node_metadata', methods=["GET"]) def all_known_nodes(): headers = {'Content-Type': 'application/octet-stream'} if this_node.known_nodes.checksum is NO_KNOWN_NODES: return Response(b"", headers=headers, status=204) known_nodes_bytestring = this_node.bytestring_of_known_nodes() signature = this_node.stamp(known_nodes_bytestring) return Response(bytes(signature) + known_nodes_bytestring, headers=headers) @rest_app.route('/node_metadata', methods=["POST"]) def node_metadata_exchange(): # If these nodes already have the same fleet state, no exchange is necessary. learner_fleet_state = request.args.get('fleet') if learner_fleet_state == this_node.known_nodes.checksum: log.debug( "Learner already knew fleet state {}; doing nothing.".format( learner_fleet_state)) headers = {'Content-Type': 'application/octet-stream'} payload = this_node.known_nodes.snapshot() + bytes( FLEET_STATES_MATCH) signature = this_node.stamp(payload) return Response(bytes(signature) + payload, headers=headers) sprouts = _node_class.batch_from_bytes(request.data) for node in sprouts: this_node.remember_node(node) # TODO: What's the right status code here? 202? Different if we already knew about the node(s)? return all_known_nodes() @rest_app.route('/consider_arrangement', methods=['POST']) def consider_arrangement(): from nucypher.policy.policies import Arrangement arrangement = Arrangement.from_bytes(request.data) # TODO: Look at the expiration and figure out if we're even staking that long. 1701 with datastore.describe(PolicyArrangement, arrangement.id.hex(), writeable=True) as new_policy_arrangement: new_policy_arrangement.arrangement_id = arrangement.id.hex( ).encode() new_policy_arrangement.expiration = arrangement.expiration new_policy_arrangement.alice_verifying_key = arrangement.alice.stamp.as_umbral_pubkey( ) # TODO: Fine, we'll add the arrangement here, but if we never hear from Alice again to enact it, # we need to prune it at some point. #1700 headers = {'Content-Type': 'application/octet-stream'} # TODO: Make this a legit response #234. return Response( b"This will eventually be an actual acceptance of the arrangement.", headers=headers) @rest_app.route("/kFrag/<id_as_hex>", methods=['POST']) def set_policy(id_as_hex): """ REST endpoint for setting a kFrag. """ policy_message_kit = UmbralMessageKit.from_bytes(request.data) alices_verifying_key = policy_message_kit.sender_verifying_key alice = _alice_class.from_public_keys( verifying_key=alices_verifying_key) try: cleartext = this_node.verify_from(alice, policy_message_kit, decrypt=True) except InvalidSignature: # TODO: Perhaps we log this? Essentially 355. return Response(status_code=400) if not this_node.federated_only: # This splitter probably belongs somewhere canonical. transaction_splitter = BytestringSplitter(32) tx, kfrag_bytes = transaction_splitter(cleartext, return_remainder=True) try: # Get all of the arrangements and verify that we'll be paid. # TODO: We'd love for this part to be impossible to reduce the risk of collusion. #1274 arranged_addresses = this_node.policy_agent.fetch_arrangement_addresses_from_policy_txid( tx, timeout=this_node.synchronous_query_timeout) except TimeExhausted: # Alice didn't pay. Return response with that weird status code. this_node.suspicious_activities_witnessed['freeriders'].append( (alice, f"No transaction matching {tx}.")) return Response(status=402) this_node_has_been_arranged = this_node.checksum_address in arranged_addresses if not this_node_has_been_arranged: this_node.suspicious_activities_witnessed['freeriders'].append( (alice, f"The transaction {tx} does not list me as a Worker - it lists {arranged_addresses}." )) return Response(status=402) else: _tx = NO_BLOCKCHAIN_CONNECTION kfrag_bytes = cleartext kfrag = KFrag.from_bytes(kfrag_bytes) if not kfrag.verify(signing_pubkey=alices_verifying_key): raise InvalidSignature("{} is invalid".format(kfrag)) with datastore.describe(PolicyArrangement, id_as_hex, writeable=True) as policy_arrangement: if not policy_arrangement.alice_verifying_key == alice.stamp.as_umbral_pubkey( ): raise alice.SuspiciousActivity policy_arrangement.kfrag = kfrag # TODO: Sign the arrangement here. #495 return "" # TODO: Return A 200, with whatever policy metadata. @rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"]) def revoke_arrangement(id_as_hex): """ REST endpoint for revoking/deleting a KFrag from a node. """ from nucypher.policy.collections import Revocation revocation = Revocation.from_bytes(request.data) log.info("Received revocation: {} -- for arrangement {}".format( bytes(revocation).hex(), id_as_hex)) # Check that the request is the same for the provided revocation if not id_as_hex == revocation.arrangement_id.hex(): log.debug("Couldn't identify an arrangement with id {}".format( id_as_hex)) return Response(status_code=400) try: with datastore.describe(PolicyArrangement, id_as_hex, writeable=True) as policy_arrangement: if revocation.verify_signature( policy_arrangement.alice_verifying_key): policy_arrangement.delete() except (DatastoreTransactionError, InvalidSignature) as e: log.debug("Exception attempting to revoke: {}".format(e)) return Response( response='KFrag not found or revocation signature is invalid.', status=404) else: log.info("KFrag successfully removed.") return Response(response='KFrag deleted!', status=200) @rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"]) def reencrypt_via_rest(id_as_hex): # Get Policy Arrangement try: arrangement_id = binascii.unhexlify(id_as_hex) except (binascii.Error, TypeError): return Response(response=b'Invalid arrangement ID', status=405) try: # Get KFrag # TODO: Yeah, well, what if this arrangement hasn't been enacted? 1702 with datastore.describe(PolicyArrangement, id_as_hex) as policy_arrangement: kfrag = policy_arrangement.kfrag alice_verifying_key = policy_arrangement.alice_verifying_key except RecordNotFound: return Response(response=arrangement_id, status=404) # Get Work Order from nucypher.policy.collections import WorkOrder # Avoid circular import alice_address = canonical_address_from_umbral_key(alice_verifying_key) work_order_payload = request.data work_order = WorkOrder.from_rest_payload( arrangement_id=arrangement_id, rest_payload=work_order_payload, ursula=this_node, alice_address=alice_address) log.info( f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}" ) # Re-encrypt response = this_node._reencrypt( kfrag=kfrag, work_order=work_order, alice_verifying_key=alice_verifying_key) # Now, Ursula saves this workorder to her database... # Note: we give the work order a random ID to store it under. with datastore.describe(Workorder, str(uuid.uuid4()), writeable=True) as new_workorder: new_workorder.arrangement_id = work_order.arrangement_id new_workorder.bob_verifying_key = work_order.bob.stamp.as_umbral_pubkey( ) new_workorder.bob_signature = work_order.receipt_signature headers = {'Content-Type': 'application/octet-stream'} return Response(headers=headers, response=response) @rest_app.route('/treasure_map/<treasure_map_id>') def provide_treasure_map(treasure_map_id): headers = {'Content-Type': 'application/octet-stream'} treasure_map_index = bytes.fromhex(treasure_map_id) try: treasure_map = this_node.treasure_maps[treasure_map_index] response = Response(bytes(treasure_map), headers=headers) log.info("{} providing TreasureMap {}".format( this_node.nickname, treasure_map_id)) except KeyError: log.info("{} doesn't have requested TreasureMap {}".format( this_node.stamp, treasure_map_id)) response = Response( "No Treasure Map with ID {}".format(treasure_map_id), status=404, headers=headers) return response @rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST']) def receive_treasure_map(treasure_map_id): # TODO: Any of the codepaths that trigger 4xx Responses here are also SuspiciousActivity. if not this_node.federated_only: from nucypher.policy.collections import SignedTreasureMap as _MapClass else: from nucypher.policy.collections import TreasureMap as _MapClass try: treasure_map = _MapClass.from_bytes( bytes_representation=request.data, verify=True) except _MapClass.InvalidSignature: log.info("Bad TreasureMap HRAC Signature; not storing {}".format( treasure_map_id)) return Response("This TreasureMap's HRAC is not properly signed.", status=401) treasure_map_index = bytes.fromhex(treasure_map_id) # First let's see if we already have this map. try: previously_saved_map = this_node.treasure_maps[treasure_map_index] except KeyError: pass # We don't have the map. We'll validate and perhaps save it. else: if previously_saved_map == treasure_map: return Response("Already have this map.", status=303) # Otherwise, if it's a different map with the same ID, we move on to validation. if treasure_map.public_id() == treasure_map_id: do_store = True else: return Response("Can't save a TreasureMap with this ID from you.", status=409) if do_store and not this_node.federated_only: alice_checksum_address = this_node.policy_agent.contract.functions.getPolicyOwner( treasure_map._hrac[:16]).call() do_store = treasure_map.verify_blockchain_signature( checksum_address=alice_checksum_address) if do_store: log.info("{} storing TreasureMap {}".format( this_node, treasure_map_id)) this_node.treasure_maps[treasure_map_index] = treasure_map return Response(bytes(treasure_map), status=202) else: log.info( "Bad TreasureMap ID; not storing {}".format(treasure_map_id)) return Response("This TreasureMap doesn't match a paid Policy.", status=402) @rest_app.route('/status/', methods=['GET']) def status(): if request.args.get('json'): payload = this_node.abridged_node_details() response = jsonify(payload) return response else: headers = {"Content-Type": "text/html", "charset": "utf-8"} previous_states = list( reversed(this_node.known_nodes.states.values()))[:5] # Mature every known node before rendering. for node in this_node.known_nodes: node.mature() try: content = status_template.render( this_node=this_node, known_nodes=this_node.known_nodes, previous_states=previous_states, domains=serving_domains, version=nucypher.__version__, checksum_address=this_node.checksum_address) except Exception as e: log.debug("Template Rendering Exception: ".format(str(e))) raise TemplateError(str(e)) from e return Response(response=content, headers=headers) return rest_app, datastore