def __setup_node_storage(self, node_storage=None) -> None: if self.dev_mode: node_storage = ForgetfulNodeStorage(registry=self.registry, federated_only=self.federated_only) elif not node_storage: node_storage = LocalFileBasedNodeStorage(registry=self.registry, config_root=self.config_root, federated_only=self.federated_only) self.node_storage = node_storage
def __setup_node_storage(self, node_storage=None) -> None: if self.dev_mode: node_storage = ForgetfulNodeStorage( blockchain=self.blockchain, federated_only=self.federated_only) elif not node_storage: node_storage = LocalFileBasedNodeStorage( blockchain=self.blockchain, federated_only=self.federated_only, config_root=self.config_root) self.node_storage = node_storage
def __init__(self, registry: Optional['BaseContractRegistry'] = None, eth_provider_uri: Optional[str] = None, storage: Optional['NodeStorage'] = None, *args, **kwargs): self.registry = registry self.eth_provider_uri = eth_provider_uri self.storage = storage or ForgetfulNodeStorage( ) # for certificate storage
def _setup_node_storage(self, node_storage=None) -> None: # TODO: Disables node metadata persistence.. # if self.dev_mode: # node_storage = ForgetfulNodeStorage(registry=self.registry, federated_only=self.federated_only) # TODO: Forcibly clears the filesystem of any stored node metadata and certificates... local_node_storage = LocalFileBasedNodeStorage( registry=self.registry, config_root=self.config_root, federated_only=self.federated_only) local_node_storage.clear() self.log.info( f'Cleared peer metadata from {local_node_storage.root_dir}') # TODO: Always sets up nodes for in-memory node metadata storage node_storage = ForgetfulNodeStorage(registry=self.registry, federated_only=self.federated_only) self.node_storage = node_storage
def __init__( self, # Base config_root: str = None, config_file_location: str = None, # Mode dev_mode: bool = False, federated_only: bool = False, # Identity is_me: bool = True, checksum_public_address: str = None, crypto_power: CryptoPower = None, # Keyring keyring: NucypherKeyring = None, keyring_dir: str = None, # Learner learn_on_same_thread: bool = False, abort_on_learning_error: bool = False, start_learning_now: bool = True, # REST rest_host: str = None, rest_port: int = None, # TLS tls_curve: EllipticCurve = None, certificate: Certificate = None, # Network domains: Set[str] = None, interface_signature: Signature = None, network_middleware: RestMiddleware = None, # Node Storage known_nodes: set = None, node_storage: NodeStorage = None, reload_metadata: bool = True, save_metadata: bool = True, # Blockchain poa: bool = False, provider_uri: str = None, # Registry registry_source: str = None, registry_filepath: str = None, import_seed_registry: bool = False # TODO: needs cleanup ) -> None: # Logs self.log = Logger(self.__class__.__name__) # # REST + TLS (Ursula) # self.rest_host = rest_host or self.DEFAULT_REST_HOST default_port = (self.DEFAULT_DEVELOPMENT_REST_PORT if dev_mode else self.DEFAULT_REST_PORT) self.rest_port = rest_port or default_port self.tls_curve = tls_curve or self.__DEFAULT_TLS_CURVE self.certificate = certificate self.interface_signature = interface_signature self.crypto_power = crypto_power # # Keyring # self.keyring = keyring or NO_KEYRING_ATTACHED self.keyring_dir = keyring_dir or UNINITIALIZED_CONFIGURATION # Contract Registry if import_seed_registry is True: registry_source = self.REGISTRY_SOURCE if not os.path.isfile(registry_source): message = "Seed contract registry does not exist at path {}.".format( registry_filepath) self.log.debug(message) raise RuntimeError(message) self.__registry_source = registry_source or self.REGISTRY_SOURCE self.registry_filepath = registry_filepath or UNINITIALIZED_CONFIGURATION # # Configuration # self.config_file_location = config_file_location or UNINITIALIZED_CONFIGURATION self.config_root = UNINITIALIZED_CONFIGURATION # # Mode # self.federated_only = federated_only self.__dev_mode = dev_mode if self.__dev_mode: self.__temp_dir = UNINITIALIZED_CONFIGURATION self.node_storage = ForgetfulNodeStorage( federated_only=federated_only, character_class=self.__class__) else: self.__temp_dir = LIVE_CONFIGURATION self.config_root = config_root or DEFAULT_CONFIG_ROOT self._cache_runtime_filepaths() self.node_storage = node_storage or LocalFileBasedNodeStorage( federated_only=federated_only, config_root=self.config_root) # Domains self.domains = domains or {self.DEFAULT_DOMAIN} # # Identity # self.is_me = is_me self.checksum_public_address = checksum_public_address if self.is_me is True or dev_mode is True: # Self if self.checksum_public_address and dev_mode is False: self.attach_keyring() self.network_middleware = network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS( ) else: # Stranger self.node_storage = STRANGER_CONFIGURATION self.keyring_dir = STRANGER_CONFIGURATION self.keyring = STRANGER_CONFIGURATION self.network_middleware = STRANGER_CONFIGURATION if network_middleware: raise self.ConfigurationError( "Cannot configure a stranger to use network middleware.") # # Learner # self.learn_on_same_thread = learn_on_same_thread self.abort_on_learning_error = abort_on_learning_error self.start_learning_now = start_learning_now self.save_metadata = save_metadata self.reload_metadata = reload_metadata self.__fleet_state = FleetStateTracker() known_nodes = known_nodes or set() if known_nodes: self.known_nodes._nodes.update( {node.checksum_public_address: node for node in known_nodes}) self.known_nodes.record_fleet_state( ) # TODO: Does this call need to be here? # # Blockchain # self.poa = poa self.provider_uri = provider_uri or self.DEFAULT_PROVIDER_URI self.blockchain = NO_BLOCKCHAIN_CONNECTION self.accounts = NO_BLOCKCHAIN_CONNECTION self.token_agent = NO_BLOCKCHAIN_CONNECTION self.miner_agent = NO_BLOCKCHAIN_CONNECTION self.policy_agent = NO_BLOCKCHAIN_CONNECTION # # Development Mode # if dev_mode: # Ephemeral dev settings self.abort_on_learning_error = True self.save_metadata = False self.reload_metadata = False # Generate one-time alphanumeric development password alphabet = string.ascii_letters + string.digits password = ''.join(secrets.choice(alphabet) for _ in range(32)) # Auto-initialize self.initialize(password=password, import_registry=import_seed_registry)
class NodeConfiguration(ABC): """ 'Sideways Engagement' of Character classes; a reflection of input parameters. """ # Abstract _NAME = NotImplemented _CHARACTER_CLASS = NotImplemented CONFIG_FILENAME = NotImplemented DEFAULT_CONFIG_FILE_LOCATION = NotImplemented # Mode DEFAULT_OPERATING_MODE = 'decentralized' # Domains DEFAULT_DOMAIN = GLOBAL_DOMAIN # Serializers NODE_SERIALIZER = binascii.hexlify NODE_DESERIALIZER = binascii.unhexlify # System __CONFIG_FILE_EXT = '.config' __CONFIG_FILE_DESERIALIZER = json.loads TEMP_CONFIGURATION_DIR_PREFIX = "nucypher-tmp-" # Blockchain DEFAULT_PROVIDER_URI = 'tester://pyevm' # Registry __REGISTRY_NAME = 'contract_registry.json' REGISTRY_SOURCE = os.path.join( BASE_DIR, __REGISTRY_NAME) # TODO: #461 Where will this be hosted? # Rest + TLS DEFAULT_REST_HOST = '127.0.0.1' DEFAULT_REST_PORT = 9151 DEFAULT_DEVELOPMENT_REST_PORT = 10151 __DEFAULT_TLS_CURVE = ec.SECP384R1 __DEFAULT_NETWORK_MIDDLEWARE_CLASS = RestMiddleware class ConfigurationError(RuntimeError): pass class InvalidConfiguration(ConfigurationError): pass def __init__( self, # Base config_root: str = None, config_file_location: str = None, # Mode dev_mode: bool = False, federated_only: bool = False, # Identity is_me: bool = True, checksum_public_address: str = None, crypto_power: CryptoPower = None, # Keyring keyring: NucypherKeyring = None, keyring_dir: str = None, # Learner learn_on_same_thread: bool = False, abort_on_learning_error: bool = False, start_learning_now: bool = True, # REST rest_host: str = None, rest_port: int = None, # TLS tls_curve: EllipticCurve = None, certificate: Certificate = None, # Network domains: Set[str] = None, interface_signature: Signature = None, network_middleware: RestMiddleware = None, # Node Storage known_nodes: set = None, node_storage: NodeStorage = None, reload_metadata: bool = True, save_metadata: bool = True, # Blockchain poa: bool = False, provider_uri: str = None, # Registry registry_source: str = None, registry_filepath: str = None, import_seed_registry: bool = False # TODO: needs cleanup ) -> None: # Logs self.log = Logger(self.__class__.__name__) # # REST + TLS (Ursula) # self.rest_host = rest_host or self.DEFAULT_REST_HOST default_port = (self.DEFAULT_DEVELOPMENT_REST_PORT if dev_mode else self.DEFAULT_REST_PORT) self.rest_port = rest_port or default_port self.tls_curve = tls_curve or self.__DEFAULT_TLS_CURVE self.certificate = certificate self.interface_signature = interface_signature self.crypto_power = crypto_power # # Keyring # self.keyring = keyring or NO_KEYRING_ATTACHED self.keyring_dir = keyring_dir or UNINITIALIZED_CONFIGURATION # Contract Registry if import_seed_registry is True: registry_source = self.REGISTRY_SOURCE if not os.path.isfile(registry_source): message = "Seed contract registry does not exist at path {}.".format( registry_filepath) self.log.debug(message) raise RuntimeError(message) self.__registry_source = registry_source or self.REGISTRY_SOURCE self.registry_filepath = registry_filepath or UNINITIALIZED_CONFIGURATION # # Configuration # self.config_file_location = config_file_location or UNINITIALIZED_CONFIGURATION self.config_root = UNINITIALIZED_CONFIGURATION # # Mode # self.federated_only = federated_only self.__dev_mode = dev_mode if self.__dev_mode: self.__temp_dir = UNINITIALIZED_CONFIGURATION self.node_storage = ForgetfulNodeStorage( federated_only=federated_only, character_class=self.__class__) else: self.__temp_dir = LIVE_CONFIGURATION self.config_root = config_root or DEFAULT_CONFIG_ROOT self._cache_runtime_filepaths() self.node_storage = node_storage or LocalFileBasedNodeStorage( federated_only=federated_only, config_root=self.config_root) # Domains self.domains = domains or {self.DEFAULT_DOMAIN} # # Identity # self.is_me = is_me self.checksum_public_address = checksum_public_address if self.is_me is True or dev_mode is True: # Self if self.checksum_public_address and dev_mode is False: self.attach_keyring() self.network_middleware = network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS( ) else: # Stranger self.node_storage = STRANGER_CONFIGURATION self.keyring_dir = STRANGER_CONFIGURATION self.keyring = STRANGER_CONFIGURATION self.network_middleware = STRANGER_CONFIGURATION if network_middleware: raise self.ConfigurationError( "Cannot configure a stranger to use network middleware.") # # Learner # self.learn_on_same_thread = learn_on_same_thread self.abort_on_learning_error = abort_on_learning_error self.start_learning_now = start_learning_now self.save_metadata = save_metadata self.reload_metadata = reload_metadata self.__fleet_state = FleetStateTracker() known_nodes = known_nodes or set() if known_nodes: self.known_nodes._nodes.update( {node.checksum_public_address: node for node in known_nodes}) self.known_nodes.record_fleet_state( ) # TODO: Does this call need to be here? # # Blockchain # self.poa = poa self.provider_uri = provider_uri or self.DEFAULT_PROVIDER_URI self.blockchain = NO_BLOCKCHAIN_CONNECTION self.accounts = NO_BLOCKCHAIN_CONNECTION self.token_agent = NO_BLOCKCHAIN_CONNECTION self.miner_agent = NO_BLOCKCHAIN_CONNECTION self.policy_agent = NO_BLOCKCHAIN_CONNECTION # # Development Mode # if dev_mode: # Ephemeral dev settings self.abort_on_learning_error = True self.save_metadata = False self.reload_metadata = False # Generate one-time alphanumeric development password alphabet = string.ascii_letters + string.digits password = ''.join(secrets.choice(alphabet) for _ in range(32)) # Auto-initialize self.initialize(password=password, import_registry=import_seed_registry) def __call__(self, *args, **kwargs): return self.produce(*args, **kwargs) @classmethod def generate(cls, password: str, no_registry: bool, *args, **kwargs) -> 'UrsulaConfiguration': """Shortcut: Hook-up a new initial installation and write configuration file to the disk""" ursula_config = cls(dev_mode=False, is_me=True, *args, **kwargs) ursula_config.__write(password=password, no_registry=no_registry) return ursula_config def __write(self, password: str, no_registry: bool): _new_installation_path = self.initialize(password=password, import_registry=no_registry) _configuration_filepath = self.to_configuration_file( filepath=self.config_file_location) def cleanup(self) -> None: if self.__dev_mode: self.__temp_dir.cleanup() @property def dev_mode(self): return self.__dev_mode @property def known_nodes(self): return self.__fleet_state def connect_to_blockchain(self, recompile_contracts: bool = False): if self.federated_only: raise NodeConfiguration.ConfigurationError( "Cannot connect to blockchain in federated mode") self.blockchain = Blockchain.connect(provider_uri=self.provider_uri, compile=recompile_contracts, poa=self.poa) self.accounts = self.blockchain.interface.w3.eth.accounts self.log.debug("Established connection to provider {}".format( self.blockchain.interface.provider_uri)) def connect_to_contracts(self) -> None: """Initialize contract agency and set them on config""" self.token_agent = NucypherTokenAgent(blockchain=self.blockchain) self.miner_agent = MinerAgent(blockchain=self.blockchain) self.policy_agent = PolicyAgent(blockchain=self.blockchain) self.log.debug("Established connection to nucypher contracts") def read_known_nodes(self): known_nodes = self.node_storage.all(federated_only=self.federated_only) known_nodes = { node.checksum_public_address: node for node in known_nodes } self.known_nodes._nodes.update(known_nodes) self.known_nodes.record_fleet_state() return self.known_nodes def forget_nodes(self) -> None: self.node_storage.clear() message = "Removed all stored node node metadata and certificates" self.log.debug(message) def destroy(self, force: bool = False, logs: bool = True) -> None: # TODO: Further confirm this is a nucypher dir first! (in-depth measure) if logs is True or force: shutil.rmtree(USER_LOG_DIR, ignore_errors=True) try: shutil.rmtree(self.config_root, ignore_errors=force) except FileNotFoundError: raise FileNotFoundError("No such directory {}".format( self.config_root)) def generate_parameters(self, **overrides) -> dict: merged_parameters = { **self.static_payload, **self.dynamic_payload, **overrides } non_init_params = ('config_root', 'poa', 'provider_uri') character_init_params = filter(lambda t: t[0] not in non_init_params, merged_parameters.items()) return dict(character_init_params) def produce(self, **overrides): """Initialize a new character instance and return it.""" merged_parameters = self.generate_parameters(**overrides) character = self._CHARACTER_CLASS(**merged_parameters) return character @staticmethod def _read_configuration_file(filepath: str) -> dict: try: with open(filepath, 'r') as file: raw_contents = file.read() payload = NodeConfiguration.__CONFIG_FILE_DESERIALIZER( raw_contents) except FileNotFoundError as e: raise # TODO: Do we need better exception handling here? return payload @classmethod def from_configuration_file(cls, filepath: str = None, **overrides) -> 'NodeConfiguration': """Initialize a NodeConfiguration from a JSON file.""" from nucypher.config.storages import NodeStorage node_storage_subclasses = { storage._name: storage for storage in NodeStorage.__subclasses__() } if filepath is None: filepath = cls.DEFAULT_CONFIG_FILE_LOCATION # Read from disk payload = cls._read_configuration_file(filepath=filepath) # Initialize NodeStorage subclass from file (sub-configuration) storage_payload = payload['node_storage'] storage_type = storage_payload[NodeStorage._TYPE_LABEL] storage_class = node_storage_subclasses[storage_type] node_storage = storage_class.from_payload( payload=storage_payload, # character_class=cls._CHARACTER_CLASS, # TODO: Do not pass this here - Always Use Ursula federated_only=payload['federated_only'], serializer=cls.NODE_SERIALIZER, deserializer=cls.NODE_DESERIALIZER) # Deserialize domains to UTF-8 bytestrings domains = list(domain.encode() for domain in payload['domains']) payload.update(dict(node_storage=node_storage, domains=domains)) # Filter out Nones from overrides to detect, well, overrides overrides = {k: v for k, v in overrides.items() if v is not None} # Instantiate from merged params node_configuration = cls(**{**payload, **overrides}) return node_configuration def to_configuration_file(self, filepath: str = None) -> str: """Write the static_payload to a JSON file.""" if filepath is None: filename = '{}{}'.format(self._NAME.lower(), self.__CONFIG_FILE_EXT) filepath = os.path.join(self.config_root, filename) payload = self.static_payload del payload['is_me'] # TODO # Serialize domains domains = list(str(d) for d in self.domains) # Save node connection data payload.update( dict(node_storage=self.node_storage.payload(), domains=domains)) with open(filepath, 'w') as config_file: config_file.write(json.dumps(payload, indent=4)) return filepath def validate(self, config_root: str, no_registry=False) -> bool: # Top-level if not os.path.exists(config_root): raise self.ConfigurationError( 'No configuration directory found at {}.'.format(config_root)) # Sub-paths filepaths = self.runtime_filepaths if no_registry: del filepaths['registry_filepath'] for field, path in filepaths.items(): if not os.path.exists(path): message = 'Missing configuration file or directory: {}.' if 'registry' in path: message += ' Did you mean to pass --federated-only?' raise NodeConfiguration.InvalidConfiguration( message.format(path)) return True @property def static_payload(self) -> dict: """Exported static configuration values for initializing Ursula""" payload = dict( config_root=self.config_root, # Identity is_me=self.is_me, federated_only=self.federated_only, checksum_public_address=self.checksum_public_address, keyring_dir=self.keyring_dir, # Behavior domains=self.domains, # From Set learn_on_same_thread=self.learn_on_same_thread, abort_on_learning_error=self.abort_on_learning_error, start_learning_now=self.start_learning_now, save_metadata=self.save_metadata, ) if not self.federated_only: payload.update(dict(provider_uri=self.provider_uri, poa=self.poa)) return payload @property def dynamic_payload(self, **overrides) -> dict: """Exported dynamic configuration values for initializing Ursula""" if self.reload_metadata: known_nodes = self.node_storage.all( federated_only=self.federated_only) known_nodes = { node.checksum_public_address: node for node in known_nodes } self.known_nodes._nodes.update(known_nodes) self.known_nodes.record_fleet_state() payload = dict(network_middleware=self.network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS(), known_nodes=self.known_nodes, node_storage=self.node_storage, crypto_power_ups=self.derive_node_power_ups() or None) if not self.federated_only: self.connect_to_blockchain(recompile_contracts=False) payload.update(blockchain=self.blockchain) if overrides: self.log.debug( "Overrides supplied to dynamic payload for {}".format( self.__class__.__name__)) payload.update(overrides) return payload @property def runtime_filepaths(self): filepaths = dict(config_root=self.config_root, keyring_dir=self.keyring_dir, registry_filepath=self.registry_filepath) return filepaths @classmethod def generate_runtime_filepaths(cls, config_root: str) -> dict: """Dynamically generate paths based on configuration root directory""" filepaths = dict( config_root=config_root, config_file_location=os.path.join(config_root, cls.CONFIG_FILENAME), keyring_dir=os.path.join(config_root, 'keyring'), registry_filepath=os.path.join(config_root, NodeConfiguration.__REGISTRY_NAME)) return filepaths def _cache_runtime_filepaths(self) -> None: """Generate runtime filepaths and cache them on the config object""" filepaths = self.generate_runtime_filepaths( config_root=self.config_root) for field, filepath in filepaths.items(): if getattr(self, field) is UNINITIALIZED_CONFIGURATION: setattr(self, field, filepath) def derive_node_power_ups(self) -> List[CryptoPowerUp]: power_ups = list() if self.is_me and not self.dev_mode: for power_class in self._CHARACTER_CLASS._default_crypto_powerups: power_up = self.keyring.derive_crypto_power(power_class) power_ups.append(power_up) return power_ups def initialize( self, password: str, import_registry: bool = True, ) -> str: """Initialize a new configuration.""" # # Create Config Root # if self.__dev_mode: self.__temp_dir = TemporaryDirectory( prefix=self.TEMP_CONFIGURATION_DIR_PREFIX) self.config_root = self.__temp_dir.name else: try: os.mkdir(self.config_root, mode=0o755) except FileExistsError: if os.listdir(self.config_root): message = "There are existing files located at {}".format( self.config_root) raise self.ConfigurationError(message) except FileNotFoundError: os.makedirs(self.config_root, mode=0o755) # # Create Config Subdirectories # self._cache_runtime_filepaths() try: # Node Storage self.node_storage.initialize() # Keyring if not self.dev_mode: os.mkdir( self.keyring_dir, mode=0o700) # keyring TODO: Keyring backend entry point self.write_keyring(password=password) # Registry if import_registry and not self.federated_only: self.write_registry( output_filepath=self.registry_filepath, # type: str source=self.__registry_source, # type: str blank=import_registry) # type: bool except FileExistsError: existing_paths = [ os.path.join(self.config_root, f) for f in os.listdir(self.config_root) ] message = "There are pre-existing files at {}: {}".format( self.config_root, existing_paths) self.log.critical(message) raise NodeConfiguration.ConfigurationError(message) if not self.__dev_mode: self.validate(config_root=self.config_root, no_registry=import_registry or self.federated_only) # Success message = "Created nucypher installation files at {}".format( self.config_root) self.log.debug(message) return self.config_root def attach_keyring(self, checksum_address: str = None, *args, **kwargs) -> None: if self.keyring is not NO_KEYRING_ATTACHED: if self.keyring.checksum_address != (checksum_address or self.checksum_public_address): raise self.ConfigurationError( "There is already a keyring attached to this configuration." ) return if (checksum_address or self.checksum_public_address) is None: raise self.ConfigurationError( "No account specified to unlock keyring") self.keyring = NucypherKeyring( keyring_root=self.keyring_dir, # type: str account=checksum_address or self.checksum_public_address, # type: str *args, **kwargs) def write_keyring(self, password: str, **generation_kwargs) -> NucypherKeyring: self.keyring = NucypherKeyring.generate(password=password, keyring_root=self.keyring_dir, **generation_kwargs) # Operating mode switch TODO: #466 if self.federated_only: self.checksum_public_address = self.keyring.federated_address else: self.checksum_public_address = self.keyring.checksum_address return self.keyring def write_registry(self, output_filepath: str = None, source: str = None, force: bool = False, blank=False) -> str: if force and os.path.isfile(output_filepath): raise self.ConfigurationError( 'There is an existing file at the registry output_filepath {}'. format(output_filepath)) output_filepath = output_filepath or self.registry_filepath source = source or self.REGISTRY_SOURCE if not blank and not self.dev_mode: # Validate Registry with open(source, 'r') as registry_file: try: json.loads(registry_file.read()) except JSONDecodeError: message = "The registry source {} is not valid JSON".format( source) self.log.critical(message) raise self.ConfigurationError(message) else: self.log.debug( "Source registry {} is valid JSON".format(source)) else: self.log.warn("Writing blank registry") open(output_filepath, 'w').close() # write blank self.log.debug( "Successfully wrote registry to {}".format(output_filepath)) return output_filepath
def make_rest_app( db_filepath: str, this_node, serving_domains, log=Logger("http-application-layer") ) -> Tuple: forgetful_node_storage = ForgetfulNodeStorage(federated_only=this_node.federated_only) from nucypher.keystore import keystore from nucypher.keystore.db import Base from sqlalchemy.engine import create_engine log.info("Starting datastore {}".format(db_filepath)) # See: https://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#connect-strings if db_filepath: db_uri = f'sqlite:///{db_filepath}' else: db_uri = 'sqlite://' # TODO: Is this a sane default? See #667 engine = create_engine(db_uri) Base.metadata.create_all(engine) datastore = keystore.KeyStore(engine) db_engine = engine from nucypher.characters.lawful import Alice, Ursula _alice_class = Alice _node_class = Ursula rest_app = Flask("ursula-service") @rest_app.route("/public_information") def public_information(): """ REST endpoint for public keys and address. """ response = Response( response=bytes(this_node), mimetype='application/octet-stream') return response @rest_app.route('/node_metadata', methods=["GET"]) def all_known_nodes(): headers = {'Content-Type': 'application/octet-stream'} if this_node.known_nodes.checksum is NO_KNOWN_NODES: return Response(b"", headers=headers, status=204) known_nodes_bytestring = this_node.bytestring_of_known_nodes() signature = this_node.stamp(known_nodes_bytestring) return Response(bytes(signature) + known_nodes_bytestring, headers=headers) @rest_app.route('/node_metadata', methods=["POST"]) def node_metadata_exchange(): # If these nodes already have the same fleet state, no exchange is necessary. learner_fleet_state = request.args.get('fleet') if learner_fleet_state == this_node.known_nodes.checksum: log.debug("Learner already knew fleet state {}; doing nothing.".format(learner_fleet_state)) headers = {'Content-Type': 'application/octet-stream'} payload = this_node.known_nodes.snapshot() + bytes(FLEET_STATES_MATCH) signature = this_node.stamp(payload) return Response(bytes(signature) + payload, headers=headers) sprouts = _node_class.batch_from_bytes(request.data, registry=this_node.registry) # TODO: This logic is basically repeated in learn_from_teacher_node and remember_node. # Let's find a better way. #555 for node in sprouts: @crosstown_traffic() def learn_about_announced_nodes(): if node in this_node.known_nodes: if node.timestamp <= this_node.known_nodes[node.checksum_address].timestamp: return node.mature() try: node.verify_node(this_node.network_middleware.client, registry=this_node.registry, ) # Suspicion except node.SuspiciousActivity as e: # TODO: Include data about caller? # TODO: Account for possibility that stamp, rather than interface, was bad. # TODO: Maybe also record the bytes representation separately to disk? message = f"Suspicious Activity about {node}: {str(e)}. Announced via REST." log.warn(message) this_node.suspicious_activities_witnessed['vladimirs'].append(node) except NodeSeemsToBeDown as e: # This is a rather odd situation - this node *just* contacted us and asked to be verified. Where'd it go? Maybe a NAT problem? log.info(f"Node announced itself to us just now, but seems to be down: {node}. Response was {e}.") log.debug(f"Phantom node certificate: {node.certificate}") # Async Sentinel except Exception as e: log.critical(f"This exception really needs to be handled differently: {e}") raise # Believable else: log.info("Learned about previously unknown node: {}".format(node)) this_node.remember_node(node) # TODO: Record new fleet state # Cleanup finally: forgetful_node_storage.forget() # TODO: What's the right status code here? 202? Different if we already knew about the node? return all_known_nodes() @rest_app.route('/consider_arrangement', methods=['POST']) def consider_arrangement(): from nucypher.policy.policies import Arrangement arrangement = Arrangement.from_bytes(request.data) with ThreadedSession(db_engine) as session: new_policy_arrangement = datastore.add_policy_arrangement( arrangement.expiration.datetime(), id=arrangement.id.hex().encode(), alice_verifying_key=arrangement.alice.stamp, session=session, ) # TODO: Make the rest of this logic actually work - do something here # to decide if this Arrangement is worth accepting. headers = {'Content-Type': 'application/octet-stream'} # TODO: Make this a legit response #234. return Response(b"This will eventually be an actual acceptance of the arrangement.", headers=headers) @rest_app.route("/kFrag/<id_as_hex>", methods=['POST']) def set_policy(id_as_hex): """ REST endpoint for setting a kFrag. TODO: Instead of taking a Request, use the apistar typing system to type a payload and validate / split it. TODO: Validate that the kfrag being saved is pursuant to an approved Policy (see #121). """ policy_message_kit = UmbralMessageKit.from_bytes(request.data) alices_verifying_key = policy_message_kit.sender_verifying_key alice = _alice_class.from_public_keys(verifying_key=alices_verifying_key) try: cleartext = this_node.verify_from(alice, policy_message_kit, decrypt=True) except InvalidSignature: # TODO: Perhaps we log this? return Response(status_code=400) kfrag = KFrag.from_bytes(cleartext) if not kfrag.verify(signing_pubkey=alices_verifying_key): raise InvalidSignature("{} is invalid".format(kfrag)) with ThreadedSession(db_engine) as session: datastore.attach_kfrag_to_saved_arrangement( alice, id_as_hex, kfrag, session=session) # TODO: Sign the arrangement here. #495 return "" # TODO: Return A 200, with whatever policy metadata. @rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"]) def revoke_arrangement(id_as_hex): """ REST endpoint for revoking/deleting a KFrag from a node. """ from nucypher.policy.collections import Revocation revocation = Revocation.from_bytes(request.data) log.info("Received revocation: {} -- for arrangement {}".format(bytes(revocation).hex(), id_as_hex)) try: with ThreadedSession(db_engine) as session: # Verify the Notice was signed by Alice policy_arrangement = datastore.get_policy_arrangement( id_as_hex.encode(), session=session) alice_pubkey = UmbralPublicKey.from_bytes( policy_arrangement.alice_verifying_key.key_data) # Check that the request is the same for the provided revocation if id_as_hex != revocation.arrangement_id.hex(): log.debug("Couldn't identify an arrangement with id {}".format(id_as_hex)) return Response(status_code=400) elif revocation.verify_signature(alice_pubkey): datastore.del_policy_arrangement( id_as_hex.encode(), session=session) except (NotFound, InvalidSignature) as e: log.debug("Exception attempting to revoke: {}".format(e)) return Response(response='KFrag not found or revocation signature is invalid.', status=404) else: log.info("KFrag successfully removed.") return Response(response='KFrag deleted!', status=200) @rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"]) def reencrypt_via_rest(id_as_hex): # Get Policy Arrangement try: arrangement_id = binascii.unhexlify(id_as_hex) except (binascii.Error, TypeError): return Response(response=b'Invalid arrangement ID', status=405) try: with ThreadedSession(db_engine) as session: arrangement = datastore.get_policy_arrangement(arrangement_id=id_as_hex.encode(), session=session) except NotFound: return Response(response=arrangement_id, status=404) # Get KFrag kfrag = KFrag.from_bytes(arrangement.kfrag) # Get Work Order from nucypher.policy.collections import WorkOrder # Avoid circular import alice_verifying_key_bytes = arrangement.alice_verifying_key.key_data alice_verifying_key = UmbralPublicKey.from_bytes(alice_verifying_key_bytes) alice_address = canonical_address_from_umbral_key(alice_verifying_key) work_order_payload = request.data work_order = WorkOrder.from_rest_payload(arrangement_id=arrangement_id, rest_payload=work_order_payload, ursula=this_node, alice_address=alice_address) log.info(f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}") # Re-encrypt response = this_node._reencrypt(kfrag=kfrag, work_order=work_order, alice_verifying_key=alice_verifying_key) # Now, Ursula saves this workorder to her database... with ThreadedSession(db_engine): this_node.datastore.save_workorder(bob_verifying_key=bytes(work_order.bob.stamp), bob_signature=bytes(work_order.receipt_signature), arrangement_id=work_order.arrangement_id) headers = {'Content-Type': 'application/octet-stream'} return Response(headers=headers, response=response) @rest_app.route('/treasure_map/<treasure_map_id>') def provide_treasure_map(treasure_map_id): headers = {'Content-Type': 'application/octet-stream'} treasure_map_index = bytes.fromhex(treasure_map_id) try: treasure_map = this_node.treasure_maps[treasure_map_index] response = Response(bytes(treasure_map), headers=headers) log.info("{} providing TreasureMap {}".format(this_node.nickname, treasure_map_id)) except KeyError: log.info("{} doesn't have requested TreasureMap {}".format(this_node.stamp, treasure_map_id)) response = Response("No Treasure Map with ID {}".format(treasure_map_id), status=404, headers=headers) return response @rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST']) def receive_treasure_map(treasure_map_id): from nucypher.policy.collections import TreasureMap try: treasure_map = TreasureMap.from_bytes(bytes_representation=request.data, verify=True) except TreasureMap.InvalidSignature: do_store = False else: do_store = treasure_map.public_id() == treasure_map_id if do_store: log.info("{} storing TreasureMap {}".format(this_node, treasure_map_id)) # TODO 341 - what if we already have this TreasureMap? treasure_map_index = bytes.fromhex(treasure_map_id) this_node.treasure_maps[treasure_map_index] = treasure_map return Response(bytes(treasure_map), status=202) else: # TODO: Make this a proper 500 or whatever. log.info("Bad TreasureMap ID; not storing {}".format(treasure_map_id)) assert False @rest_app.route('/status/', methods=['GET']) def status(): if request.args.get('json'): payload = this_node.abridged_node_details() response = jsonify(payload) return response else: headers = {"Content-Type": "text/html", "charset": "utf-8"} previous_states = list(reversed(this_node.known_nodes.states.values()))[:5] # Mature every known node before rendering. for node in this_node.known_nodes: node.mature() try: content = status_template.render(this_node=this_node, known_nodes=this_node.known_nodes, previous_states=previous_states, domains=serving_domains, version=nucypher.__version__, checksum_address=this_node.checksum_address) except Exception as e: log.debug("Template Rendering Exception: ".format(str(e))) raise TemplateError(str(e)) from e return Response(response=content, headers=headers) return rest_app, datastore
def _make_rest_app(datastore: Datastore, this_node, serving_domain: str, log: Logger) -> Tuple[Flask, Datastore]: forgetful_node_storage = ForgetfulNodeStorage(federated_only=this_node.federated_only) # FIXME: Seems unused from nucypher.characters.lawful import Alice, Ursula _alice_class = Alice _node_class = Ursula rest_app = Flask("ursula-service") rest_app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD_CONTENT_LENGTH @rest_app.route("/public_information") def public_information(): """ REST endpoint for public keys and address. """ response = Response( response=bytes(this_node), mimetype='application/octet-stream') return response @rest_app.route("/ping", methods=['POST']) def ping(): """ Asks this node: "Can you access my public information endpoint"? """ try: requesting_ursula = Ursula.from_bytes(request.data, registry=this_node.registry) requesting_ursula.mature() except ValueError: # (ValueError) return Response({'error': 'Invalid Ursula'}, status=400) else: initiator_address, initiator_port = tuple(requesting_ursula.rest_interface) # Compare requester and posted Ursula information request_address = request.environ['REMOTE_ADDR'] if request_address != initiator_address: return Response({'error': 'Suspicious origin address'}, status=400) # # Make a Sandwich # try: # Fetch and store initiator's teacher certificate. certificate = this_node.network_middleware.get_certificate(host=initiator_address, port=initiator_port) certificate_filepath = this_node.node_storage.store_node_certificate(certificate=certificate) requesting_ursula_bytes = this_node.network_middleware.client.node_information(host=initiator_address, port=initiator_port, certificate_filepath=certificate_filepath) except NodeSeemsToBeDown: return Response({'error': 'Unreachable node'}, status=400) # ... toasted except InvalidNodeCertificate: return Response({'error': 'Invalid TLS certificate - missing checksum address'}, status=400) # ... invalid # Compare the results of the outer POST with the inner GET... yum if requesting_ursula_bytes == request.data: return Response(status=200) else: return Response({'error': 'Suspicious node'}, status=400) @rest_app.route('/node_metadata', methods=["GET"]) def all_known_nodes(): headers = {'Content-Type': 'application/octet-stream'} if this_node.known_nodes.checksum is NO_KNOWN_NODES: return Response(b"", headers=headers, status=204) known_nodes_bytestring = this_node.bytestring_of_known_nodes() signature = this_node.stamp(known_nodes_bytestring) return Response(bytes(signature) + known_nodes_bytestring, headers=headers) @rest_app.route('/node_metadata', methods=["POST"]) def node_metadata_exchange(): # If these nodes already have the same fleet state, no exchange is necessary. learner_fleet_state = request.args.get('fleet') if learner_fleet_state == this_node.known_nodes.checksum: log.debug("Learner already knew fleet state {}; doing nothing.".format(learner_fleet_state)) headers = {'Content-Type': 'application/octet-stream'} payload = this_node.known_nodes.snapshot() + bytes(FLEET_STATES_MATCH) signature = this_node.stamp(payload) return Response(bytes(signature) + payload, headers=headers) sprouts = _node_class.batch_from_bytes(request.data) for node in sprouts: this_node.remember_node(node) # TODO: What's the right status code here? 202? Different if we already knew about the node(s)? return all_known_nodes() @rest_app.route('/consider_arrangement', methods=['POST']) def consider_arrangement(): from nucypher.policy.policies import Arrangement arrangement = Arrangement.from_bytes(request.data) # TODO: Look at the expiration and figure out if we're even staking that long. 1701 with datastore.describe(PolicyArrangement, arrangement.id.hex(), writeable=True) as new_policy_arrangement: new_policy_arrangement.arrangement_id = arrangement.id.hex().encode() new_policy_arrangement.expiration = arrangement.expiration new_policy_arrangement.alice_verifying_key = arrangement.alice.stamp.as_umbral_pubkey() # TODO: Fine, we'll add the arrangement here, but if we never hear from Alice again to enact it, # we need to prune it at some point. #1700 headers = {'Content-Type': 'application/octet-stream'} # TODO: Make this a legit response #234. return Response(b"This will eventually be an actual acceptance of the arrangement.", headers=headers) @rest_app.route("/kFrag/<id_as_hex>", methods=['POST']) def set_policy(id_as_hex): """ REST endpoint for setting a kFrag. """ policy_message_kit = UmbralMessageKit.from_bytes(request.data) alices_verifying_key = policy_message_kit.sender_verifying_key alice = _alice_class.from_public_keys(verifying_key=alices_verifying_key) try: cleartext = this_node.verify_from(alice, policy_message_kit, decrypt=True) except InvalidSignature: # TODO: Perhaps we log this? Essentially 355. return Response(status_code=400) if not this_node.federated_only: # This splitter probably belongs somewhere canonical. transaction_splitter = BytestringSplitter(32) tx, kfrag_bytes = transaction_splitter(cleartext, return_remainder=True) try: # Get all of the arrangements and verify that we'll be paid. # TODO: We'd love for this part to be impossible to reduce the risk of collusion. #1274 arranged_addresses = this_node.policy_agent.fetch_arrangement_addresses_from_policy_txid(tx, timeout=this_node.synchronous_query_timeout) except TimeExhausted: # Alice didn't pay. Return response with that weird status code. this_node.suspicious_activities_witnessed['freeriders'].append((alice, f"No transaction matching {tx}.")) return Response(status=402) this_node_has_been_arranged = this_node.checksum_address in arranged_addresses if not this_node_has_been_arranged: this_node.suspicious_activities_witnessed['freeriders'].append((alice, f"The transaction {tx} does not list me as a Worker - it lists {arranged_addresses}.")) return Response(status=402) else: _tx = NO_BLOCKCHAIN_CONNECTION kfrag_bytes = cleartext kfrag = KFrag.from_bytes(kfrag_bytes) if not kfrag.verify(signing_pubkey=alices_verifying_key): raise InvalidSignature("{} is invalid".format(kfrag)) with datastore.describe(PolicyArrangement, id_as_hex, writeable=True) as policy_arrangement: if not policy_arrangement.alice_verifying_key == alice.stamp.as_umbral_pubkey(): raise alice.SuspiciousActivity policy_arrangement.kfrag = kfrag # TODO: Sign the arrangement here. #495 return "" # TODO: Return A 200, with whatever policy metadata. @rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"]) def revoke_arrangement(id_as_hex): """ REST endpoint for revoking/deleting a KFrag from a node. """ from nucypher.policy.collections import Revocation revocation = Revocation.from_bytes(request.data) log.info("Received revocation: {} -- for arrangement {}".format(bytes(revocation).hex(), id_as_hex)) # Check that the request is the same for the provided revocation if not id_as_hex == revocation.arrangement_id.hex(): log.debug("Couldn't identify an arrangement with id {}".format(id_as_hex)) return Response(status_code=400) try: with datastore.describe(PolicyArrangement, id_as_hex, writeable=True) as policy_arrangement: if revocation.verify_signature(policy_arrangement.alice_verifying_key): policy_arrangement.delete() except (DatastoreTransactionError, InvalidSignature) as e: log.debug("Exception attempting to revoke: {}".format(e)) return Response(response='KFrag not found or revocation signature is invalid.', status=404) else: log.info("KFrag successfully removed.") return Response(response='KFrag deleted!', status=200) @rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"]) def reencrypt_via_rest(id_as_hex): # Get Policy Arrangement try: arrangement_id = binascii.unhexlify(id_as_hex) except (binascii.Error, TypeError): return Response(response=b'Invalid arrangement ID', status=405) try: # Get KFrag # TODO: Yeah, well, what if this arrangement hasn't been enacted? 1702 with datastore.describe(PolicyArrangement, id_as_hex) as policy_arrangement: kfrag = policy_arrangement.kfrag alice_verifying_key = policy_arrangement.alice_verifying_key except RecordNotFound: return Response(response=arrangement_id, status=404) # Get Work Order from nucypher.policy.collections import WorkOrder # Avoid circular import alice_address = canonical_address_from_umbral_key(alice_verifying_key) work_order_payload = request.data work_order = WorkOrder.from_rest_payload(arrangement_id=arrangement_id, rest_payload=work_order_payload, ursula=this_node, alice_address=alice_address) log.info(f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}") # Re-encrypt response = this_node._reencrypt(kfrag=kfrag, work_order=work_order, alice_verifying_key=alice_verifying_key) # Now, Ursula saves this workorder to her database... # Note: we give the work order a random ID to store it under. with datastore.describe(Workorder, str(uuid.uuid4()), writeable=True) as new_workorder: new_workorder.arrangement_id = work_order.arrangement_id new_workorder.bob_verifying_key = work_order.bob.stamp.as_umbral_pubkey() new_workorder.bob_signature = work_order.receipt_signature headers = {'Content-Type': 'application/octet-stream'} return Response(headers=headers, response=response) @rest_app.route('/treasure_map/<treasure_map_id>') def provide_treasure_map(treasure_map_id): headers = {'Content-Type': 'application/octet-stream'} treasure_map_index = bytes.fromhex(treasure_map_id) try: treasure_map = this_node.treasure_maps[treasure_map_index] response = Response(bytes(treasure_map), headers=headers) log.info("{} providing TreasureMap {}".format(this_node.nickname, treasure_map_id)) except KeyError: log.info("{} doesn't have requested TreasureMap {}".format(this_node.stamp, treasure_map_id)) response = Response("No Treasure Map with ID {}".format(treasure_map_id), status=404, headers=headers) return response @rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST']) def receive_treasure_map(treasure_map_id): # TODO: Any of the codepaths that trigger 4xx Responses here are also SuspiciousActivity. if not this_node.federated_only: from nucypher.policy.collections import SignedTreasureMap as _MapClass else: from nucypher.policy.collections import TreasureMap as _MapClass try: treasure_map = _MapClass.from_bytes(bytes_representation=request.data, verify=True) except _MapClass.InvalidSignature: log.info("Bad TreasureMap HRAC Signature; not storing {}".format(treasure_map_id)) return Response("This TreasureMap's HRAC is not properly signed.", status=401) treasure_map_index = bytes.fromhex(treasure_map_id) # First let's see if we already have this map. try: previously_saved_map = this_node.treasure_maps[treasure_map_index] except KeyError: pass # We don't have the map. We'll validate and perhaps save it. else: if previously_saved_map == treasure_map: return Response("Already have this map.", status=303) # Otherwise, if it's a different map with the same ID, we move on to validation. if treasure_map.public_id() == treasure_map_id: do_store = True else: return Response("Can't save a TreasureMap with this ID from you.", status=409) if do_store and not this_node.federated_only: alice_checksum_address = this_node.policy_agent.contract.functions.getPolicyOwner( treasure_map._hrac[:16]).call() do_store = treasure_map.verify_blockchain_signature(checksum_address=alice_checksum_address) if do_store: log.info("{} storing TreasureMap {}".format(this_node, treasure_map_id)) this_node.treasure_maps[treasure_map_index] = treasure_map return Response(bytes(treasure_map), status=202) else: log.info("Bad TreasureMap ID; not storing {}".format(treasure_map_id)) return Response("This TreasureMap doesn't match a paid Policy.", status=402) @rest_app.route('/status/', methods=['GET']) def status(): if request.args.get('json'): payload = this_node.abridged_node_details() response = jsonify(payload) return response else: headers = {"Content-Type": "text/html", "charset": "utf-8"} previous_states = list(reversed(this_node.known_nodes.states.values()))[:5] # Mature every known node before rendering. for node in this_node.known_nodes: node.mature() try: content = status_template.render(this_node=this_node, known_nodes=this_node.known_nodes, previous_states=previous_states, domain=serving_domain, version=nucypher.__version__, checksum_address=this_node.checksum_address) except Exception as e: log.debug("Template Rendering Exception: ".format(str(e))) raise TemplateError(str(e)) from e return Response(response=content, headers=headers) return rest_app
class TestInMemoryNodeStorage(BaseTestNodeStorageBackends): storage_backend = ForgetfulNodeStorage( character_class=BaseTestNodeStorageBackends.character_class, federated_only=BaseTestNodeStorageBackends.federated_only) storage_backend.initialize()
def make_rest_app( db_filepath: str, network_middleware: RestMiddleware, federated_only: bool, treasure_map_tracker: dict, node_tracker: 'FleetStateTracker', node_bytes_caster: Callable, work_order_tracker: list, node_nickname: str, node_recorder: Callable, stamp: SignatureStamp, verifier: Callable, suspicious_activity_tracker: dict, serving_domains, log=Logger("http-application-layer")) -> Tuple: forgetful_node_storage = ForgetfulNodeStorage( federated_only=federated_only) from nucypher.keystore import keystore from nucypher.keystore.db import Base from sqlalchemy.engine import create_engine log.info("Starting datastore {}".format(db_filepath)) # See: https://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#connect-strings if db_filepath: db_uri = f'sqlite:///{db_filepath}' else: db_uri = 'sqlite://' # TODO: Is this a sane default? See #667 engine = create_engine(db_uri) Base.metadata.create_all(engine) datastore = keystore.KeyStore(engine) db_engine = engine from nucypher.characters.lawful import Alice, Ursula _alice_class = Alice _node_class = Ursula rest_app = Flask("ursula-service") @rest_app.route("/public_information") def public_information(): """ REST endpoint for public keys and address.. """ response = Response(response=node_bytes_caster(), mimetype='application/octet-stream') return response @rest_app.route('/node_metadata', methods=["GET"]) def all_known_nodes(): headers = {'Content-Type': 'application/octet-stream'} if node_tracker.checksum is NO_KNOWN_NODES: return Response(b"", headers=headers, status=204) payload = node_tracker.snapshot() ursulas_as_vbytes = (VariableLengthBytestring(n) for n in node_tracker) ursulas_as_bytes = bytes().join(bytes(u) for u in ursulas_as_vbytes) ursulas_as_bytes += VariableLengthBytestring(node_bytes_caster()) payload += ursulas_as_bytes signature = stamp(payload) return Response(bytes(signature) + payload, headers=headers) @rest_app.route('/node_metadata', methods=["POST"]) def node_metadata_exchange(): # If these nodes already have the same fleet state, no exchange is necessary. learner_fleet_state = request.args.get('fleet') if learner_fleet_state == node_tracker.checksum: log.debug( "Learner already knew fleet state {}; doing nothing.".format( learner_fleet_state)) headers = {'Content-Type': 'application/octet-stream'} payload = node_tracker.snapshot() + bytes(FLEET_STATES_MATCH) signature = stamp(payload) return Response(bytes(signature) + payload, headers=headers) nodes = _node_class.batch_from_bytes( request.data, federated_only=federated_only) # TODO: 466 # TODO: This logic is basically repeated in learn_from_teacher_node and remember_node. # Let's find a better way. #555 for node in nodes: if GLOBAL_DOMAIN not in serving_domains: if not serving_domains.intersection(node.serving_domains): continue # This node is not serving any of our domains. if node in node_tracker: if node.timestamp <= node_tracker[ node.checksum_public_address].timestamp: continue @crosstown_traffic() def learn_about_announced_nodes(): try: certificate_filepath = forgetful_node_storage.store_node_certificate( certificate=node.certificate) node.verify_node( network_middleware, accept_federated_only=federated_only, # TODO: 466 certificate_filepath=certificate_filepath) # Suspicion except node.SuspiciousActivity: # TODO: Include data about caller? # TODO: Account for possibility that stamp, rather than interface, was bad. # TODO: Maybe also record the bytes representation separately to disk? message = "Suspicious Activity: Discovered node with bad signature: {}. Announced via REST." log.warn(message) suspicious_activity_tracker['vladimirs'].append(node) # Async Sentinel except Exception as e: log.critical(str(e)) raise # Believable else: log.info( "Learned about previously unknown node: {}".format( node)) node_recorder(node) # TODO: Record new fleet state # Cleanup finally: forgetful_node_storage.forget() # TODO: What's the right status code here? 202? Different if we already knew about the node? return all_known_nodes() @rest_app.route('/consider_arrangement', methods=['POST']) def consider_arrangement(): from nucypher.policy.models import Arrangement arrangement = Arrangement.from_bytes(request.data) with ThreadedSession(db_engine) as session: new_policy_arrangement = datastore.add_policy_arrangement( arrangement.expiration.datetime(), id=arrangement.id.hex().encode(), alice_pubkey_sig=arrangement.alice.stamp, session=session, ) # TODO: Make the rest of this logic actually work - do something here # to decide if this Arrangement is worth accepting. headers = {'Content-Type': 'application/octet-stream'} # TODO: Make this a legit response #234. return Response( b"This will eventually be an actual acceptance of the arrangement.", headers=headers) @rest_app.route("/kFrag/<id_as_hex>", methods=['POST']) def set_policy(id_as_hex): """ REST endpoint for setting a kFrag. TODO: Instead of taking a Request, use the apistar typing system to type a payload and validate / split it. TODO: Validate that the kfrag being saved is pursuant to an approved Policy (see #121). """ policy_message_kit = UmbralMessageKit.from_bytes(request.data) alices_verifying_key = policy_message_kit.sender_pubkey_sig alice = _alice_class.from_public_keys( {SigningPower: alices_verifying_key}) try: cleartext = verifier(alice, policy_message_kit, decrypt=True) except InvalidSignature: # TODO: Perhaps we log this? return Response(status_code=400) kfrag = KFrag.from_bytes(cleartext) if not kfrag.verify(signing_pubkey=alices_verifying_key): raise InvalidSignature("{} is invalid".format(kfrag)) with ThreadedSession(db_engine) as session: datastore.attach_kfrag_to_saved_arrangement(alice, id_as_hex, kfrag, session=session) # TODO: Sign the arrangement here. #495 return "" # TODO: Return A 200, with whatever policy metadata. @rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"]) def revoke_arrangement(id_as_hex): """ REST endpoint for revoking/deleting a KFrag from a node. """ from nucypher.policy.models import Revocation revocation = Revocation.from_bytes(request.data) log.info("Received revocation: {} -- for arrangement {}".format( bytes(revocation).hex(), id_as_hex)) try: with ThreadedSession(db_engine) as session: # Verify the Notice was signed by Alice policy_arrangement = datastore.get_policy_arrangement( id_as_hex.encode(), session=session) alice_pubkey = UmbralPublicKey.from_bytes( policy_arrangement.alice_pubkey_sig.key_data) # Check that the request is the same for the provided revocation if id_as_hex != revocation.arrangement_id.hex(): log.debug( "Couldn't identify an arrangement with id {}".format( id_as_hex)) return Response(status_code=400) elif revocation.verify_signature(alice_pubkey): datastore.del_policy_arrangement(id_as_hex.encode(), session=session) except (NotFound, InvalidSignature) as e: log.debug("Exception attempting to revoke: {}".format(e)) return Response( response='KFrag not found or revocation signature is invalid.', status=404) else: log.info("KFrag successfully removed.") return Response(response='KFrag deleted!', status=200) @rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"]) def reencrypt_via_rest(id_as_hex): from nucypher.policy.models import WorkOrder # Avoid circular import arrangement_id = binascii.unhexlify(id_as_hex) work_order = WorkOrder.from_rest_payload(arrangement_id, request.data) log.info("Work Order from {}, signed {}".format( work_order.bob, work_order.receipt_signature)) with ThreadedSession(db_engine) as session: policy_arrangement = datastore.get_policy_arrangement( arrangement_id=id_as_hex.encode(), session=session) kfrag_bytes = policy_arrangement.kfrag # Careful! :-) verifying_key_bytes = policy_arrangement.alice_pubkey_sig.key_data # TODO: Push this to a lower level. Perhaps to Ursula character? #619 kfrag = KFrag.from_bytes(kfrag_bytes) alices_verifying_key = UmbralPublicKey.from_bytes(verifying_key_bytes) cfrag_byte_stream = b"" alices_address = canonical_address_from_umbral_key( alices_verifying_key) if not alices_address == work_order.alice_address: message = f"This Bob ({work_order.bob}) sent an Alice's ETH address " \ f"({work_order.alice_address}) that doesn't match " \ f"the one I have ({alices_address})." raise SuspiciousActivity(message) bob_pubkey = work_order.bob.stamp.as_umbral_pubkey() if not work_order.alice_address_signature.verify( message=alices_address, verifying_key=bob_pubkey): message = f"This Bob ({work_order.bob}) sent an invalid signature of Alice's ETH address" raise InvalidSignature(message) # This is Bob's signature of Alice's verifying key as ETH address. alice_address_signature = bytes(work_order.alice_address_signature) for capsule, capsule_signature in zip(work_order.capsules, work_order.capsule_signatures): # This is the capsule signed by Bob capsule_signature = bytes(capsule_signature) # Ursula signs on top of it. Now both are committed to the same capsule. # She signs Alice's address too. ursula_signature = stamp(capsule_signature + alice_address_signature) capsule.set_correctness_keys(verifying=alices_verifying_key) cfrag = pre.reencrypt(kfrag, capsule, metadata=bytes(ursula_signature)) log.info(f"Re-encrypting for {capsule}, made {cfrag}.") signature = stamp(bytes(cfrag) + bytes(capsule)) cfrag_byte_stream += VariableLengthBytestring(cfrag) + signature # TODO: Put this in Ursula's datastore work_order_tracker.append(work_order) headers = {'Content-Type': 'application/octet-stream'} return Response(response=cfrag_byte_stream, headers=headers) @rest_app.route('/treasure_map/<treasure_map_id>') def provide_treasure_map(treasure_map_id): headers = {'Content-Type': 'application/octet-stream'} treasure_map_bytes = keccak_digest(binascii.unhexlify(treasure_map_id)) try: treasure_map = treasure_map_tracker[treasure_map_bytes] response = Response(bytes(treasure_map), headers=headers) log.info("{} providing TreasureMap {}".format( node_nickname, treasure_map_id)) except KeyError: log.info("{} doesn't have requested TreasureMap {}".format( stamp, treasure_map_id)) response = Response( "No Treasure Map with ID {}".format(treasure_map_id), status=404, headers=headers) return response @rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST']) def receive_treasure_map(treasure_map_id): from nucypher.policy.models import TreasureMap try: treasure_map = TreasureMap.from_bytes( bytes_representation=request.data, verify=True) except TreasureMap.InvalidSignature: do_store = False else: do_store = treasure_map.public_id() == treasure_map_id if do_store: log.info("{} storing TreasureMap {}".format( stamp, treasure_map_id)) # # # # # TODO: Now that the DHT is retired, let's do this another way. # self.dht_server.set_now(binascii.unhexlify(treasure_map_id), # constants.BYTESTRING_IS_TREASURE_MAP + bytes(treasure_map)) # # # # # TODO 341 - what if we already have this TreasureMap? treasure_map_tracker[keccak_digest( binascii.unhexlify(treasure_map_id))] = treasure_map return Response(bytes(treasure_map), status=202) else: # TODO: Make this a proper 500 or whatever. log.info( "Bad TreasureMap ID; not storing {}".format(treasure_map_id)) assert False @rest_app.route('/status') def status(): # TODO: Seems very strange to deserialize *this node* when we can just pass it in. # Might be a sign that we need to rethnk this composition. headers = {"Content-Type": "text/html", "charset": "utf-8"} this_node = _node_class.from_bytes(node_bytes_caster(), federated_only=federated_only) previous_states = list(reversed(node_tracker.states.values()))[:5] try: content = status_template.render(this_node=this_node, known_nodes=node_tracker, previous_states=previous_states) except Exception as e: log.debug("Template Rendering Exception: ".format(str(e))) raise TemplateError(str(e)) from e return Response(response=content, headers=headers) return rest_app, datastore
def from_seed_and_stake_info( cls, seed_uri: str, federated_only: bool, minimum_stake: int = 0, checksum_address: str = None, # TODO: Why is this unused? network_middleware: RestMiddleware = None, *args, **kwargs) -> 'Ursula': # # WARNING: xxx Poison xxx # Let's learn what we can about the ... "seednode". # if network_middleware is None: network_middleware = RestMiddleware() host, port, checksum_address = parse_node_uri(seed_uri) # Fetch the hosts TLS certificate and read the common name certificate = network_middleware.get_certificate(host=host, port=port) real_host = certificate.subject.get_attributes_for_oid( NameOID.COMMON_NAME)[0].value temp_node_storage = ForgetfulNodeStorage(federated_only=federated_only) certificate_filepath = temp_node_storage.store_node_certificate( certificate=certificate) # Load the host as a potential seed node potential_seed_node = cls.from_rest_url( host=real_host, port=port, network_middleware=network_middleware, certificate_filepath=certificate_filepath, federated_only=True, *args, **kwargs) # TODO: 466 potential_seed_node.certificate_filepath = certificate_filepath if checksum_address: # Ensure this is the specific node we expected if not checksum_address == potential_seed_node.checksum_public_address: template = "This seed node has a different wallet address: {} (expected {}). Are you sure this is a seednode?" raise potential_seed_node.SuspiciousActivity( template.format( potential_seed_node.checksum_public_address, checksum_address)) # Check the node's stake (optional) if minimum_stake > 0: # TODO: check the blockchain to verify that address has more then minimum_stake. #511 raise NotImplementedError("Stake checking is not implemented yet.") # Verify the node's TLS certificate try: potential_seed_node.verify_node( network_middleware=network_middleware, accept_federated_only=federated_only, certificate_filepath=certificate_filepath) except potential_seed_node.InvalidNode: raise # TODO: What if our seed node fails verification? # OK - everyone get out temp_node_storage.forget() return potential_seed_node
def make_rest_app( db_filepath: str, this_node, serving_domains, log=Logger("http-application-layer") ) -> Tuple: forgetful_node_storage = ForgetfulNodeStorage(federated_only=this_node.federated_only) from nucypher.datastore import datastore from nucypher.datastore.db import Base from sqlalchemy.engine import create_engine log.info("Starting datastore {}".format(db_filepath)) # See: https://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#connect-strings if db_filepath: db_uri = f'sqlite:///{db_filepath}' else: db_uri = 'sqlite://' # TODO: Is this a sane default? See #667 engine = create_engine(db_uri) Base.metadata.create_all(engine) datastore = datastore.Datastore(engine) db_engine = engine from nucypher.characters.lawful import Alice, Ursula _alice_class = Alice _node_class = Ursula rest_app = Flask("ursula-service") rest_app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD_CONTENT_LENGTH @rest_app.route("/public_information") def public_information(): """ REST endpoint for public keys and address. """ response = Response( response=bytes(this_node), mimetype='application/octet-stream') return response @rest_app.route("/ping", methods=['POST']) def ping(): """ Asks this node: "Can you access my public information endpoint"? """ try: requesting_ursula = Ursula.from_bytes(request.data, registry=this_node.registry) requesting_ursula.mature() except ValueError: # (ValueError) return Response({'error': 'Invalid Ursula'}, status=400) else: initiator_address, initiator_port = tuple(requesting_ursula.rest_interface) # Compare requester and posted Ursula information request_address = request.environ['REMOTE_ADDR'] if request_address != initiator_address: return Response({'error': 'Suspicious origin address'}, status=400) # # Make a Sandwich # try: # Fetch and store initiator's teacher certificate. certificate = this_node.network_middleware.get_certificate(host=initiator_address, port=initiator_port) certificate_filepath = this_node.node_storage.store_node_certificate(certificate=certificate) requesting_ursula_bytes = this_node.network_middleware.client.node_information(host=initiator_address, port=initiator_port, certificate_filepath=certificate_filepath) except NodeSeemsToBeDown: return Response({'error': 'Unreachable node'}, status=400) # ... toasted # Compare the results of the outer POST with the inner GET... yum if requesting_ursula_bytes == request.data: return Response(status=200) else: return Response({'error': 'Suspicious node'}, status=400) @rest_app.route('/node_metadata', methods=["GET"]) def all_known_nodes(): headers = {'Content-Type': 'application/octet-stream'} if this_node.known_nodes.checksum is NO_KNOWN_NODES: return Response(b"", headers=headers, status=204) known_nodes_bytestring = this_node.bytestring_of_known_nodes() signature = this_node.stamp(known_nodes_bytestring) return Response(bytes(signature) + known_nodes_bytestring, headers=headers) @rest_app.route('/node_metadata', methods=["POST"]) def node_metadata_exchange(): # If these nodes already have the same fleet state, no exchange is necessary. learner_fleet_state = request.args.get('fleet') if learner_fleet_state == this_node.known_nodes.checksum: log.debug("Learner already knew fleet state {}; doing nothing.".format(learner_fleet_state)) headers = {'Content-Type': 'application/octet-stream'} payload = this_node.known_nodes.snapshot() + bytes(FLEET_STATES_MATCH) signature = this_node.stamp(payload) return Response(bytes(signature) + payload, headers=headers) sprouts = _node_class.batch_from_bytes(request.data, registry=this_node.registry) # TODO: This logic is basically repeated in learn_from_teacher_node and remember_node. # Let's find a better way. #555 for node in sprouts: @crosstown_traffic() def learn_about_announced_nodes(): if node in this_node.known_nodes: if node.timestamp <= this_node.known_nodes[node.checksum_address].timestamp: return node.mature() try: node.verify_node(this_node.network_middleware.client, registry=this_node.registry) # Suspicion except node.SuspiciousActivity as e: # 355 # TODO: Include data about caller? # TODO: Account for possibility that stamp, rather than interface, was bad. # TODO: Maybe also record the bytes representation separately to disk? message = f"Suspicious Activity about {node}: {str(e)}. Announced via REST." log.warn(message) this_node.suspicious_activities_witnessed['vladimirs'].append(node) except NodeSeemsToBeDown as e: # This is a rather odd situation - this node *just* contacted us and asked to be verified. Where'd it go? Maybe a NAT problem? log.info(f"Node announced itself to us just now, but seems to be down: {node}. Response was {e}.") log.debug(f"Phantom node certificate: {node.certificate}") # Async Sentinel except Exception as e: log.critical(f"This exception really needs to be handled differently: {e}") raise # Believable else: log.info("Learned about previously unknown node: {}".format(node)) this_node.remember_node(node) # TODO: Record new fleet state # Cleanup finally: forgetful_node_storage.forget() # TODO: What's the right status code here? 202? Different if we already knew about the node? return all_known_nodes() @rest_app.route('/consider_arrangement', methods=['POST']) def consider_arrangement(): from nucypher.policy.policies import Arrangement arrangement = Arrangement.from_bytes(request.data) # TODO: Look at the expiration and figure out if we're even staking that long. 1701 with ThreadedSession(db_engine) as session: new_policy_arrangement = datastore.add_policy_arrangement( arrangement.expiration.datetime(), arrangement_id=arrangement.id.hex().encode(), alice_verifying_key=arrangement.alice.stamp, session=session, ) # TODO: Fine, we'll add the arrangement here, but if we never hear from Alice again to enact it, # we need to prune it at some point. #1700 headers = {'Content-Type': 'application/octet-stream'} # TODO: Make this a legit response #234. return Response(b"This will eventually be an actual acceptance of the arrangement.", headers=headers) @rest_app.route("/kFrag/<id_as_hex>", methods=['POST']) def set_policy(id_as_hex): """ REST endpoint for setting a kFrag. """ policy_message_kit = UmbralMessageKit.from_bytes(request.data) alices_verifying_key = policy_message_kit.sender_verifying_key alice = _alice_class.from_public_keys(verifying_key=alices_verifying_key) try: cleartext = this_node.verify_from(alice, policy_message_kit, decrypt=True) except InvalidSignature: # TODO: Perhaps we log this? Essentially 355. return Response(status_code=400) if not this_node.federated_only: # This splitter probably belongs somewhere canonical. transaction_splitter = BytestringSplitter(32) tx, kfrag_bytes = transaction_splitter(cleartext, return_remainder=True) try: # Get all of the arrangements and verify that we'll be paid. # TODO: We'd love for this part to be impossible to reduce the risk of collusion. #1274 arranged_addresses = this_node.policy_agent.fetch_arrangement_addresses_from_policy_txid(tx, timeout=this_node.synchronous_query_timeout) except TimeExhausted: # Alice didn't pay. Return response with that weird status code. this_node.suspicious_activities_witnessed['freeriders'].append((alice, f"No transaction matching {tx}.")) return Response(status=402) this_node_has_been_arranged = this_node.checksum_address in arranged_addresses if not this_node_has_been_arranged: this_node.suspicious_activities_witnessed['freeriders'].append((alice, f"The transaction {tx} does not list me as a Worker - it lists {arranged_addresses}.")) return Response(status=402) else: _tx = NO_BLOCKCHAIN_CONNECTION kfrag_bytes = cleartext kfrag = KFrag.from_bytes(kfrag_bytes) if not kfrag.verify(signing_pubkey=alices_verifying_key): raise InvalidSignature("{} is invalid".format(kfrag)) with ThreadedSession(db_engine) as session: datastore.attach_kfrag_to_saved_arrangement( alice, id_as_hex, kfrag, session=session) # TODO: Sign the arrangement here. #495 return "" # TODO: Return A 200, with whatever policy metadata. @rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"]) def revoke_arrangement(id_as_hex): """ REST endpoint for revoking/deleting a KFrag from a node. """ from nucypher.policy.collections import Revocation revocation = Revocation.from_bytes(request.data) log.info("Received revocation: {} -- for arrangement {}".format(bytes(revocation).hex(), id_as_hex)) try: with ThreadedSession(db_engine) as session: # Verify the Notice was signed by Alice policy_arrangement = datastore.get_policy_arrangement( id_as_hex.encode(), session=session) alice_pubkey = UmbralPublicKey.from_bytes( policy_arrangement.alice_verifying_key.key_data) # Check that the request is the same for the provided revocation if id_as_hex != revocation.arrangement_id.hex(): log.debug("Couldn't identify an arrangement with id {}".format(id_as_hex)) return Response(status_code=400) elif revocation.verify_signature(alice_pubkey): datastore.del_policy_arrangement( id_as_hex.encode(), session=session) except (NotFound, InvalidSignature) as e: log.debug("Exception attempting to revoke: {}".format(e)) return Response(response='KFrag not found or revocation signature is invalid.', status=404) else: log.info("KFrag successfully removed.") return Response(response='KFrag deleted!', status=200) @rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"]) def reencrypt_via_rest(id_as_hex): # Get Policy Arrangement try: arrangement_id = binascii.unhexlify(id_as_hex) except (binascii.Error, TypeError): return Response(response=b'Invalid arrangement ID', status=405) try: with ThreadedSession(db_engine) as session: arrangement = datastore.get_policy_arrangement(arrangement_id=id_as_hex.encode(), session=session) except NotFound: return Response(response=arrangement_id, status=404) # Get KFrag # TODO: Yeah, well, what if this arrangement hasn't been enacted? 1702 kfrag = KFrag.from_bytes(arrangement.kfrag) # Get Work Order from nucypher.policy.collections import WorkOrder # Avoid circular import alice_verifying_key_bytes = arrangement.alice_verifying_key.key_data alice_verifying_key = UmbralPublicKey.from_bytes(alice_verifying_key_bytes) alice_address = canonical_address_from_umbral_key(alice_verifying_key) work_order_payload = request.data work_order = WorkOrder.from_rest_payload(arrangement_id=arrangement_id, rest_payload=work_order_payload, ursula=this_node, alice_address=alice_address) log.info(f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}") # Re-encrypt response = this_node._reencrypt(kfrag=kfrag, work_order=work_order, alice_verifying_key=alice_verifying_key) # Now, Ursula saves this workorder to her database... with ThreadedSession(db_engine): this_node.datastore.save_workorder(bob_verifying_key=bytes(work_order.bob.stamp), bob_signature=bytes(work_order.receipt_signature), arrangement_id=work_order.arrangement_id) headers = {'Content-Type': 'application/octet-stream'} return Response(headers=headers, response=response) @rest_app.route('/treasure_map/<treasure_map_id>') def provide_treasure_map(treasure_map_id): headers = {'Content-Type': 'application/octet-stream'} treasure_map_index = bytes.fromhex(treasure_map_id) try: treasure_map = this_node.treasure_maps[treasure_map_index] response = Response(bytes(treasure_map), headers=headers) log.info("{} providing TreasureMap {}".format(this_node.nickname, treasure_map_id)) except KeyError: log.info("{} doesn't have requested TreasureMap {}".format(this_node.stamp, treasure_map_id)) response = Response("No Treasure Map with ID {}".format(treasure_map_id), status=404, headers=headers) return response @rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST']) def receive_treasure_map(treasure_map_id): from nucypher.policy.collections import TreasureMap try: treasure_map = TreasureMap.from_bytes(bytes_representation=request.data, verify=True) except TreasureMap.InvalidSignature: do_store = False else: # TODO: If we include the policy ID in this check, does that prevent map spam? 1736 do_store = treasure_map.public_id() == treasure_map_id if do_store: log.info("{} storing TreasureMap {}".format(this_node, treasure_map_id)) # TODO 341 - what if we already have this TreasureMap? treasure_map_index = bytes.fromhex(treasure_map_id) this_node.treasure_maps[treasure_map_index] = treasure_map return Response(bytes(treasure_map), status=202) else: # TODO: Make this a proper 500 or whatever. #341 log.info("Bad TreasureMap ID; not storing {}".format(treasure_map_id)) assert False @rest_app.route('/status/', methods=['GET']) def status(): if request.args.get('json'): payload = this_node.abridged_node_details() response = jsonify(payload) return response else: headers = {"Content-Type": "text/html", "charset": "utf-8"} previous_states = list(reversed(this_node.known_nodes.states.values()))[:5] # Mature every known node before rendering. for node in this_node.known_nodes: node.mature() try: content = status_template.render(this_node=this_node, known_nodes=this_node.known_nodes, previous_states=previous_states, domains=serving_domains, version=nucypher.__version__, checksum_address=this_node.checksum_address) except Exception as e: log.debug("Template Rendering Exception: ".format(str(e))) raise TemplateError(str(e)) from e return Response(response=content, headers=headers) return rest_app, datastore
class NodeConfiguration(ABC): """ 'Sideways Engagement' of Character classes; a reflection of input parameters. """ # Abstract _NAME = NotImplemented _CHARACTER_CLASS = NotImplemented CONFIG_FILENAME = NotImplemented DEFAULT_CONFIG_FILE_LOCATION = NotImplemented # Mode DEFAULT_OPERATING_MODE = 'decentralized' # Domains DEFAULT_DOMAIN = 'goerli' # Serializers NODE_SERIALIZER = binascii.hexlify NODE_DESERIALIZER = binascii.unhexlify # System __CONFIG_FILE_EXT = '.config' __CONFIG_FILE_DESERIALIZER = json.loads TEMP_CONFIGURATION_DIR_PREFIX = "nucypher-tmp-" # Blockchain DEFAULT_PROVIDER_URI = 'http://localhost:8545' # Registry __REGISTRY_NAME = 'contract_registry.json' REGISTRY_SOURCE = os.path.join(BASE_DIR, __REGISTRY_NAME) # Rest + TLS DEFAULT_REST_HOST = '127.0.0.1' DEFAULT_REST_PORT = 9151 DEFAULT_DEVELOPMENT_REST_PORT = 10151 DEFAULT_CONTROLLER_PORT = NotImplemented __DEFAULT_TLS_CURVE = ec.SECP384R1 __DEFAULT_NETWORK_MIDDLEWARE_CLASS = RestMiddleware class ConfigurationError(RuntimeError): pass class InvalidConfiguration(ConfigurationError): pass class NoConfigurationRoot(InvalidConfiguration): pass def __init__(self, # Base config_root: str = None, config_file_location: str = None, # Mode dev_mode: bool = False, federated_only: bool = False, # Identity is_me: bool = True, checksum_address: str = None, crypto_power: CryptoPower = None, # Keyring keyring: NucypherKeyring = None, keyring_dir: str = None, # Learner learn_on_same_thread: bool = False, abort_on_learning_error: bool = False, start_learning_now: bool = True, # REST rest_host: str = None, rest_port: int = None, controller_port: int = None, # TLS tls_curve: EllipticCurve = None, certificate: Certificate = None, # Network domains: Set[str] = None, interface_signature: Signature = None, network_middleware: RestMiddleware = None, # Node Storage known_nodes: set = None, node_storage: NodeStorage = None, reload_metadata: bool = True, save_metadata: bool = True, # Blockchain poa: bool = False, provider_uri: str = None, provider_process = None, # Registry registry_source: str = None, registry_filepath: str = None, download_registry: bool = True ) -> None: # Logs self.log = Logger(self.__class__.__name__) # # REST + TLS + Web # self.controller_port = controller_port or self.DEFAULT_CONTROLLER_PORT self.rest_host = rest_host or self.DEFAULT_REST_HOST default_port = (self.DEFAULT_DEVELOPMENT_REST_PORT if dev_mode else self.DEFAULT_REST_PORT) self.rest_port = rest_port or default_port self.tls_curve = tls_curve or self.__DEFAULT_TLS_CURVE self.certificate = certificate self.interface_signature = interface_signature self.crypto_power = crypto_power # # Keyring # self.keyring = keyring or NO_KEYRING_ATTACHED self.keyring_dir = keyring_dir or UNINITIALIZED_CONFIGURATION # Contract Registry self.download_registry = download_registry self.__registry_source = registry_source or self.REGISTRY_SOURCE self.registry_filepath = registry_filepath or UNINITIALIZED_CONFIGURATION # # Configuration # self.config_file_location = config_file_location or UNINITIALIZED_CONFIGURATION self.config_root = UNINITIALIZED_CONFIGURATION # # Mode # self.federated_only = federated_only self.__dev_mode = dev_mode if self.__dev_mode: self.__temp_dir = UNINITIALIZED_CONFIGURATION self.node_storage = ForgetfulNodeStorage(federated_only=federated_only, character_class=self.__class__) else: self.__temp_dir = LIVE_CONFIGURATION self.config_root = config_root or DEFAULT_CONFIG_ROOT self._cache_runtime_filepaths() self.node_storage = node_storage or LocalFileBasedNodeStorage(federated_only=federated_only, config_root=self.config_root) # Domains self.domains = domains or {self.DEFAULT_DOMAIN} # # Identity # self.is_me = is_me self.checksum_address = checksum_address if self.is_me is True or dev_mode is True: # Self if self.checksum_address and dev_mode is False: self.attach_keyring() self.network_middleware = network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS() else: # Stranger self.node_storage = STRANGER_CONFIGURATION self.keyring_dir = STRANGER_CONFIGURATION self.keyring = STRANGER_CONFIGURATION self.network_middleware = STRANGER_CONFIGURATION if network_middleware: raise self.ConfigurationError("Cannot configure a stranger to use network middleware.") # # Learner # self.learn_on_same_thread = learn_on_same_thread self.abort_on_learning_error = abort_on_learning_error self.start_learning_now = start_learning_now self.save_metadata = save_metadata self.reload_metadata = reload_metadata self.__fleet_state = FleetStateTracker() known_nodes = known_nodes or set() if known_nodes: self.known_nodes._nodes.update({node.checksum_address: node for node in known_nodes}) self.known_nodes.record_fleet_state() # TODO: Does this call need to be here? # # Blockchain # self.poa = poa self.provider_uri = provider_uri or self.DEFAULT_PROVIDER_URI self.provider_process = provider_process or NO_BLOCKCHAIN_CONNECTION self.blockchain = NO_BLOCKCHAIN_CONNECTION.bool_value(False) self.accounts = NO_BLOCKCHAIN_CONNECTION self.token_agent = NO_BLOCKCHAIN_CONNECTION self.miner_agent = NO_BLOCKCHAIN_CONNECTION self.policy_agent = NO_BLOCKCHAIN_CONNECTION # # Development Mode # if dev_mode: # Ephemeral dev settings self.abort_on_learning_error = True self.save_metadata = False self.reload_metadata = False # Generate one-time alphanumeric development password alphabet = string.ascii_letters + string.digits password = ''.join(secrets.choice(alphabet) for _ in range(32)) # Auto-initialize self.initialize(password=password, download_registry=download_registry) def __call__(self, *args, **kwargs): return self.produce(*args, **kwargs) @classmethod def generate(cls, password: str, *args, **kwargs): """Shortcut: Hook-up a new initial installation and write configuration file to the disk""" node_config = cls(dev_mode=False, is_me=True, *args, **kwargs) node_config.__write(password=password) return node_config def __write(self, password: str): _new_installation_path = self.initialize(password=password, download_registry=self.download_registry) _configuration_filepath = self.to_configuration_file(filepath=self.config_file_location) def cleanup(self) -> None: if self.__dev_mode: self.__temp_dir.cleanup() if self.blockchain: self.blockchain.disconnect() @property def dev_mode(self): return self.__dev_mode @property def known_nodes(self): return self.__fleet_state def connect_to_blockchain(self, enode: str = None, recompile_contracts: bool = False, full_sync: bool = False) -> None: """ :param enode: ETH seednode or bootnode enode address to start learning from, i.e. 'enode://[email protected]:30303' :param recompile_contracts: Recompile all contracts on connection. :return: None """ if self.federated_only: raise NodeConfiguration.ConfigurationError("Cannot connect to blockchain in federated mode") self.blockchain = Blockchain.connect(provider_uri=self.provider_uri, compile=recompile_contracts, poa=self.poa, fetch_registry=True, provider_process=self.provider_process, sync=full_sync) # Read Ethereum Node Keyring self.accounts = self.blockchain.interface.w3.eth.accounts # Add Ethereum Peer if enode: if self.blockchain.interface.client_version == 'geth': self.blockchain.interface.w3.geth.admin.addPeer(enode) else: raise NotImplementedError def connect_to_contracts(self) -> None: """Initialize contract agency and set them on config""" self.token_agent = NucypherTokenAgent(blockchain=self.blockchain) self.miner_agent = MinerAgent(blockchain=self.blockchain) self.policy_agent = PolicyAgent(blockchain=self.blockchain) self.log.debug("Established connection to nucypher contracts") def read_known_nodes(self): known_nodes = self.node_storage.all(federated_only=self.federated_only) known_nodes = {node.checksum_address: node for node in known_nodes} self.known_nodes._nodes.update(known_nodes) self.known_nodes.record_fleet_state() return self.known_nodes def forget_nodes(self) -> None: self.node_storage.clear() message = "Removed all stored node node metadata and certificates" self.log.debug(message) def destroy(self) -> None: """Parse a node configuration and remove all associated files from the filesystem""" self.keyring.destroy() os.remove(self.config_file_location) def generate_parameters(self, **overrides) -> dict: merged_parameters = {**self.static_payload, **self.dynamic_payload, **overrides} non_init_params = ('config_root', 'poa', 'provider_uri') character_init_params = filter(lambda t: t[0] not in non_init_params, merged_parameters.items()) return dict(character_init_params) def produce(self, **overrides): """Initialize a new character instance and return it.""" merged_parameters = self.generate_parameters(**overrides) character = self._CHARACTER_CLASS(**merged_parameters) return character @staticmethod def _read_configuration_file(filepath: str) -> dict: try: with open(filepath, 'r') as file: raw_contents = file.read() payload = NodeConfiguration.__CONFIG_FILE_DESERIALIZER(raw_contents) except FileNotFoundError: raise return payload @classmethod def get_configuration_payload(cls, filepath: str = None, **overrides) -> dict: from nucypher.config.storages import NodeStorage node_storage_subclasses = {storage._name: storage for storage in NodeStorage.__subclasses__()} if filepath is None: filepath = cls.DEFAULT_CONFIG_FILE_LOCATION # Read from disk payload = cls._read_configuration_file(filepath=filepath) # Sanity check try: checksum_address = payload['checksum_address'] except KeyError: raise cls.ConfigurationError(f"No checksum address specified in configuration file {filepath}") else: if not eth_utils.is_checksum_address(checksum_address): raise cls.ConfigurationError(f"Address: '{checksum_address}', specified in {filepath} is not a valid checksum address.") # Initialize NodeStorage subclass from file (sub-configuration) storage_payload = payload['node_storage'] storage_type = storage_payload[NodeStorage._TYPE_LABEL] storage_class = node_storage_subclasses[storage_type] node_storage = storage_class.from_payload(payload=storage_payload, federated_only=payload['federated_only'], serializer=cls.NODE_SERIALIZER, deserializer=cls.NODE_DESERIALIZER) domains = set(payload['domains']) payload.update(dict(node_storage=node_storage, domains=domains)) # Filter out Nones from overrides to detect, well, overrides overrides = {k: v for k, v in overrides.items() if v is not None} payload = {**payload, **overrides} return payload @classmethod def from_configuration_file(cls, filepath: str = None, provider_process=None, **overrides) -> 'NodeConfiguration': """Initialize a NodeConfiguration from a JSON file.""" payload = cls.get_configuration_payload(filepath=filepath, **overrides) # Instantiate from merged params node_configuration = cls(config_file_location=filepath, provider_process=provider_process, **payload) return node_configuration def to_configuration_file(self, filepath: str = None) -> str: """Write the static_payload to a JSON file.""" if not filepath: filepath = os.path.join(self.config_root, self.CONFIG_FILENAME) if os.path.isfile(filepath): # Avoid overriding an existing default configuration filename = f'{self._NAME.lower()}-{self.checksum_address[:6]}{self.__CONFIG_FILE_EXT}' filepath = os.path.join(self.config_root, filename) payload = self.static_payload del payload['is_me'] # Save node connection data payload.update(dict(node_storage=self.node_storage.payload(), domains=list(self.domains))) with open(filepath, 'w') as config_file: config_file.write(json.dumps(payload, indent=4)) return filepath def validate(self, config_root: str, no_registry=False) -> bool: # Top-level if not os.path.exists(config_root): raise self.ConfigurationError('No configuration directory found at {}.'.format(config_root)) # Sub-paths filepaths = self.runtime_filepaths if no_registry: del filepaths['registry_filepath'] for field, path in filepaths.items(): if not os.path.exists(path): message = 'Missing configuration file or directory: {}.' if 'registry' in path: message += ' Did you mean to pass --federated-only?' raise NodeConfiguration.InvalidConfiguration(message.format(path)) return True @property def static_payload(self) -> dict: """Exported static configuration values for initializing Ursula""" payload = dict( config_root=self.config_root, # Identity is_me=self.is_me, federated_only=self.federated_only, checksum_address=self.checksum_address, keyring_dir=self.keyring_dir, # Behavior domains=self.domains, # From Set provider_uri=self.provider_uri, learn_on_same_thread=self.learn_on_same_thread, abort_on_learning_error=self.abort_on_learning_error, start_learning_now=self.start_learning_now, save_metadata=self.save_metadata, ) if not self.federated_only: payload.update(dict(provider_uri=self.provider_uri, poa=self.poa)) return payload @property def dynamic_payload(self, connect_to_blockchain: bool = True, **overrides) -> dict: """Exported dynamic configuration values for initializing Ursula""" if self.reload_metadata: known_nodes = self.node_storage.all(federated_only=self.federated_only) known_nodes = {node.checksum_address: node for node in known_nodes} self.known_nodes._nodes.update(known_nodes) self.known_nodes.record_fleet_state() payload = dict(network_middleware=self.network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS(), known_nodes=self.known_nodes, node_storage=self.node_storage, crypto_power_ups=self.derive_node_power_ups() or None) if not self.federated_only and connect_to_blockchain: self.connect_to_blockchain(recompile_contracts=False) payload.update(blockchain=self.blockchain) if overrides: self.log.debug("Overrides supplied to dynamic payload for {}".format(self.__class__.__name__)) payload.update(overrides) return payload @property def runtime_filepaths(self): filepaths = dict(config_root=self.config_root, keyring_dir=self.keyring_dir, registry_filepath=self.registry_filepath) return filepaths @classmethod def generate_runtime_filepaths(cls, config_root: str) -> dict: """Dynamically generate paths based on configuration root directory""" filepaths = dict(config_root=config_root, config_file_location=os.path.join(config_root, cls.CONFIG_FILENAME), keyring_dir=os.path.join(config_root, 'keyring'), registry_filepath=os.path.join(config_root, NodeConfiguration.__REGISTRY_NAME)) return filepaths def _cache_runtime_filepaths(self) -> None: """Generate runtime filepaths and cache them on the config object""" filepaths = self.generate_runtime_filepaths(config_root=self.config_root) for field, filepath in filepaths.items(): if getattr(self, field) is UNINITIALIZED_CONFIGURATION: setattr(self, field, filepath) def derive_node_power_ups(self) -> List[CryptoPowerUp]: power_ups = list() if self.is_me and not self.dev_mode: for power_class in self._CHARACTER_CLASS._default_crypto_powerups: power_up = self.keyring.derive_crypto_power(power_class) power_ups.append(power_up) return power_ups def initialize(self, password: str, download_registry: bool = True) -> str: """Initialize a new configuration and write installation files to disk.""" # # Create Base System Filepaths # if self.__dev_mode: self.__temp_dir = TemporaryDirectory(prefix=self.TEMP_CONFIGURATION_DIR_PREFIX) self.config_root = self.__temp_dir.name else: # Production Configuration try: os.mkdir(self.config_root, mode=0o755) except FileExistsError: if os.listdir(self.config_root): message = "There are existing files located at {}".format(self.config_root) self.log.debug(message) except FileNotFoundError: os.makedirs(self.config_root, mode=0o755) # Generate Installation Subdirectories self._cache_runtime_filepaths() # # Node Storage # self.node_storage.initialize() # # Keyring # if not self.dev_mode: if not os.path.isdir(self.keyring_dir): os.mkdir(self.keyring_dir, mode=0o700) # TODO: Keyring backend entry point - COS self.write_keyring(password=password) # # Registry # if download_registry and not self.federated_only: self.registry_filepath = EthereumContractRegistry.download_latest_publication() # # Verify # if not self.__dev_mode: self.validate(config_root=self.config_root, no_registry=(not download_registry) or self.federated_only) # # Success # message = "Created nucypher installation files at {}".format(self.config_root) self.log.debug(message) return self.config_root def attach_keyring(self, checksum_address: str = None, *args, **kwargs) -> None: if self.keyring is not NO_KEYRING_ATTACHED: if self.keyring.checksum_address != (checksum_address or self.checksum_address): raise self.ConfigurationError("There is already a keyring attached to this configuration.") return if (checksum_address or self.checksum_address) is None: raise self.ConfigurationError("No account specified to unlock keyring") self.keyring = NucypherKeyring(keyring_root=self.keyring_dir, # type: str account=checksum_address or self.checksum_address, # type: str *args, **kwargs) def write_keyring(self, password: str, wallet: bool = True, **generation_kwargs) -> NucypherKeyring: checksum_address = None # # Decentralized # if wallet: # Note: It is assumed the blockchain is not yet available. if not self.federated_only and not self.checksum_address: # "Casual Geth" if self.provider_process: if not os.path.exists(self.provider_process.data_dir): os.mkdir(self.provider_process.data_dir) # Get or create wallet address (geth etherbase) checksum_address = self.provider_process.ensure_account_exists(password=password) # "Formal Geth" - Manual Web3 Provider, We assume is already running and available else: self.connect_to_blockchain() if not self.blockchain.interface.client.accounts: raise self.ConfigurationError(f'Web3 provider "{self.provider_uri}" does not have any accounts') checksum_address = self.blockchain.interface.client.etherbase # Addresses read from some node keyrings (clients) are *not* returned in checksum format. checksum_address = to_checksum_address(checksum_address) # Use explicit address elif self.checksum_address: checksum_address = self.checksum_address self.keyring = NucypherKeyring.generate(password=password, keyring_root=self.keyring_dir, checksum_address=checksum_address, **generation_kwargs) # Operating mode switch if self.federated_only or not wallet: self.checksum_address = self.keyring.federated_address else: self.checksum_address = self.keyring.account return self.keyring def write_registry(self, output_filepath: str = None, source: str = None, force: bool = False, blank=False) -> str: if force and os.path.isfile(output_filepath): raise self.ConfigurationError( 'There is an existing file at the registry output_filepath {}'.format(output_filepath)) output_filepath = output_filepath or self.registry_filepath source = source or self.REGISTRY_SOURCE if not blank and not self.dev_mode: # Validate Registry with open(source, 'r') as registry_file: try: json.loads(registry_file.read()) except JSONDecodeError: message = "The registry source {} is not valid JSON".format(source) self.log.critical(message) raise self.ConfigurationError(message) else: self.log.debug("Source registry {} is valid JSON".format(source)) else: self.log.warn("Writing blank registry") open(output_filepath, 'w').close() # write blank self.log.debug("Successfully wrote registry to {}".format(output_filepath)) return output_filepath