def __setup_node_storage(self, node_storage=None) -> None:
if self.dev_mode:
node_storage = ForgetfulNodeStorage(registry=self.registry, federated_only=self.federated_only)
elif not node_storage:
node_storage = LocalFileBasedNodeStorage(registry=self.registry,
config_root=self.config_root,
federated_only=self.federated_only)
self.node_storage = node_storage
def __setup_node_storage(self, node_storage=None) -> None:
if self.dev_mode:
node_storage = ForgetfulNodeStorage(
blockchain=self.blockchain, federated_only=self.federated_only)
elif not node_storage:
node_storage = LocalFileBasedNodeStorage(
blockchain=self.blockchain,
federated_only=self.federated_only,
config_root=self.config_root)
self.node_storage = node_storage
def __init__(self,
registry: Optional['BaseContractRegistry'] = None,
eth_provider_uri: Optional[str] = None,
storage: Optional['NodeStorage'] = None,
*args,
**kwargs):
self.registry = registry
self.eth_provider_uri = eth_provider_uri
self.storage = storage or ForgetfulNodeStorage(
) # for certificate storage
def _setup_node_storage(self, node_storage=None) -> None:
# TODO: Disables node metadata persistence..
# if self.dev_mode:
# node_storage = ForgetfulNodeStorage(registry=self.registry, federated_only=self.federated_only)
# TODO: Forcibly clears the filesystem of any stored node metadata and certificates...
local_node_storage = LocalFileBasedNodeStorage(
registry=self.registry,
config_root=self.config_root,
federated_only=self.federated_only)
local_node_storage.clear()
self.log.info(
f'Cleared peer metadata from {local_node_storage.root_dir}')
# TODO: Always sets up nodes for in-memory node metadata storage
node_storage = ForgetfulNodeStorage(registry=self.registry,
federated_only=self.federated_only)
self.node_storage = node_storage
def __init__(
self,
# Base
config_root: str = None,
config_file_location: str = None,
# Mode
dev_mode: bool = False,
federated_only: bool = False,
# Identity
is_me: bool = True,
checksum_public_address: str = None,
crypto_power: CryptoPower = None,
# Keyring
keyring: NucypherKeyring = None,
keyring_dir: str = None,
# Learner
learn_on_same_thread: bool = False,
abort_on_learning_error: bool = False,
start_learning_now: bool = True,
# REST
rest_host: str = None,
rest_port: int = None,
# TLS
tls_curve: EllipticCurve = None,
certificate: Certificate = None,
# Network
domains: Set[str] = None,
interface_signature: Signature = None,
network_middleware: RestMiddleware = None,
# Node Storage
known_nodes: set = None,
node_storage: NodeStorage = None,
reload_metadata: bool = True,
save_metadata: bool = True,
# Blockchain
poa: bool = False,
provider_uri: str = None,
# Registry
registry_source: str = None,
registry_filepath: str = None,
import_seed_registry: bool = False # TODO: needs cleanup
) -> None:
# Logs
self.log = Logger(self.__class__.__name__)
#
# REST + TLS (Ursula)
#
self.rest_host = rest_host or self.DEFAULT_REST_HOST
default_port = (self.DEFAULT_DEVELOPMENT_REST_PORT
if dev_mode else self.DEFAULT_REST_PORT)
self.rest_port = rest_port or default_port
self.tls_curve = tls_curve or self.__DEFAULT_TLS_CURVE
self.certificate = certificate
self.interface_signature = interface_signature
self.crypto_power = crypto_power
#
# Keyring
#
self.keyring = keyring or NO_KEYRING_ATTACHED
self.keyring_dir = keyring_dir or UNINITIALIZED_CONFIGURATION
# Contract Registry
if import_seed_registry is True:
registry_source = self.REGISTRY_SOURCE
if not os.path.isfile(registry_source):
message = "Seed contract registry does not exist at path {}.".format(
registry_filepath)
self.log.debug(message)
raise RuntimeError(message)
self.__registry_source = registry_source or self.REGISTRY_SOURCE
self.registry_filepath = registry_filepath or UNINITIALIZED_CONFIGURATION
#
# Configuration
#
self.config_file_location = config_file_location or UNINITIALIZED_CONFIGURATION
self.config_root = UNINITIALIZED_CONFIGURATION
#
# Mode
#
self.federated_only = federated_only
self.__dev_mode = dev_mode
if self.__dev_mode:
self.__temp_dir = UNINITIALIZED_CONFIGURATION
self.node_storage = ForgetfulNodeStorage(
federated_only=federated_only, character_class=self.__class__)
else:
self.__temp_dir = LIVE_CONFIGURATION
self.config_root = config_root or DEFAULT_CONFIG_ROOT
self._cache_runtime_filepaths()
self.node_storage = node_storage or LocalFileBasedNodeStorage(
federated_only=federated_only, config_root=self.config_root)
# Domains
self.domains = domains or {self.DEFAULT_DOMAIN}
#
# Identity
#
self.is_me = is_me
self.checksum_public_address = checksum_public_address
if self.is_me is True or dev_mode is True:
# Self
if self.checksum_public_address and dev_mode is False:
self.attach_keyring()
self.network_middleware = network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS(
)
else:
# Stranger
self.node_storage = STRANGER_CONFIGURATION
self.keyring_dir = STRANGER_CONFIGURATION
self.keyring = STRANGER_CONFIGURATION
self.network_middleware = STRANGER_CONFIGURATION
if network_middleware:
raise self.ConfigurationError(
"Cannot configure a stranger to use network middleware.")
#
# Learner
#
self.learn_on_same_thread = learn_on_same_thread
self.abort_on_learning_error = abort_on_learning_error
self.start_learning_now = start_learning_now
self.save_metadata = save_metadata
self.reload_metadata = reload_metadata
self.__fleet_state = FleetStateTracker()
known_nodes = known_nodes or set()
if known_nodes:
self.known_nodes._nodes.update(
{node.checksum_public_address: node
for node in known_nodes})
self.known_nodes.record_fleet_state(
) # TODO: Does this call need to be here?
#
# Blockchain
#
self.poa = poa
self.provider_uri = provider_uri or self.DEFAULT_PROVIDER_URI
self.blockchain = NO_BLOCKCHAIN_CONNECTION
self.accounts = NO_BLOCKCHAIN_CONNECTION
self.token_agent = NO_BLOCKCHAIN_CONNECTION
self.miner_agent = NO_BLOCKCHAIN_CONNECTION
self.policy_agent = NO_BLOCKCHAIN_CONNECTION
#
# Development Mode
#
if dev_mode:
# Ephemeral dev settings
self.abort_on_learning_error = True
self.save_metadata = False
self.reload_metadata = False
# Generate one-time alphanumeric development password
alphabet = string.ascii_letters + string.digits
password = ''.join(secrets.choice(alphabet) for _ in range(32))
# Auto-initialize
self.initialize(password=password,
import_registry=import_seed_registry)
class NodeConfiguration(ABC):
"""
'Sideways Engagement' of Character classes; a reflection of input parameters.
"""
# Abstract
_NAME = NotImplemented
_CHARACTER_CLASS = NotImplemented
CONFIG_FILENAME = NotImplemented
DEFAULT_CONFIG_FILE_LOCATION = NotImplemented
# Mode
DEFAULT_OPERATING_MODE = 'decentralized'
# Domains
DEFAULT_DOMAIN = GLOBAL_DOMAIN
# Serializers
NODE_SERIALIZER = binascii.hexlify
NODE_DESERIALIZER = binascii.unhexlify
# System
__CONFIG_FILE_EXT = '.config'
__CONFIG_FILE_DESERIALIZER = json.loads
TEMP_CONFIGURATION_DIR_PREFIX = "nucypher-tmp-"
# Blockchain
DEFAULT_PROVIDER_URI = 'tester://pyevm'
# Registry
__REGISTRY_NAME = 'contract_registry.json'
REGISTRY_SOURCE = os.path.join(
BASE_DIR, __REGISTRY_NAME) # TODO: #461 Where will this be hosted?
# Rest + TLS
DEFAULT_REST_HOST = '127.0.0.1'
DEFAULT_REST_PORT = 9151
DEFAULT_DEVELOPMENT_REST_PORT = 10151
__DEFAULT_TLS_CURVE = ec.SECP384R1
__DEFAULT_NETWORK_MIDDLEWARE_CLASS = RestMiddleware
class ConfigurationError(RuntimeError):
pass
class InvalidConfiguration(ConfigurationError):
pass
def __init__(
self,
# Base
config_root: str = None,
config_file_location: str = None,
# Mode
dev_mode: bool = False,
federated_only: bool = False,
# Identity
is_me: bool = True,
checksum_public_address: str = None,
crypto_power: CryptoPower = None,
# Keyring
keyring: NucypherKeyring = None,
keyring_dir: str = None,
# Learner
learn_on_same_thread: bool = False,
abort_on_learning_error: bool = False,
start_learning_now: bool = True,
# REST
rest_host: str = None,
rest_port: int = None,
# TLS
tls_curve: EllipticCurve = None,
certificate: Certificate = None,
# Network
domains: Set[str] = None,
interface_signature: Signature = None,
network_middleware: RestMiddleware = None,
# Node Storage
known_nodes: set = None,
node_storage: NodeStorage = None,
reload_metadata: bool = True,
save_metadata: bool = True,
# Blockchain
poa: bool = False,
provider_uri: str = None,
# Registry
registry_source: str = None,
registry_filepath: str = None,
import_seed_registry: bool = False # TODO: needs cleanup
) -> None:
# Logs
self.log = Logger(self.__class__.__name__)
#
# REST + TLS (Ursula)
#
self.rest_host = rest_host or self.DEFAULT_REST_HOST
default_port = (self.DEFAULT_DEVELOPMENT_REST_PORT
if dev_mode else self.DEFAULT_REST_PORT)
self.rest_port = rest_port or default_port
self.tls_curve = tls_curve or self.__DEFAULT_TLS_CURVE
self.certificate = certificate
self.interface_signature = interface_signature
self.crypto_power = crypto_power
#
# Keyring
#
self.keyring = keyring or NO_KEYRING_ATTACHED
self.keyring_dir = keyring_dir or UNINITIALIZED_CONFIGURATION
# Contract Registry
if import_seed_registry is True:
registry_source = self.REGISTRY_SOURCE
if not os.path.isfile(registry_source):
message = "Seed contract registry does not exist at path {}.".format(
registry_filepath)
self.log.debug(message)
raise RuntimeError(message)
self.__registry_source = registry_source or self.REGISTRY_SOURCE
self.registry_filepath = registry_filepath or UNINITIALIZED_CONFIGURATION
#
# Configuration
#
self.config_file_location = config_file_location or UNINITIALIZED_CONFIGURATION
self.config_root = UNINITIALIZED_CONFIGURATION
#
# Mode
#
self.federated_only = federated_only
self.__dev_mode = dev_mode
if self.__dev_mode:
self.__temp_dir = UNINITIALIZED_CONFIGURATION
self.node_storage = ForgetfulNodeStorage(
federated_only=federated_only, character_class=self.__class__)
else:
self.__temp_dir = LIVE_CONFIGURATION
self.config_root = config_root or DEFAULT_CONFIG_ROOT
self._cache_runtime_filepaths()
self.node_storage = node_storage or LocalFileBasedNodeStorage(
federated_only=federated_only, config_root=self.config_root)
# Domains
self.domains = domains or {self.DEFAULT_DOMAIN}
#
# Identity
#
self.is_me = is_me
self.checksum_public_address = checksum_public_address
if self.is_me is True or dev_mode is True:
# Self
if self.checksum_public_address and dev_mode is False:
self.attach_keyring()
self.network_middleware = network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS(
)
else:
# Stranger
self.node_storage = STRANGER_CONFIGURATION
self.keyring_dir = STRANGER_CONFIGURATION
self.keyring = STRANGER_CONFIGURATION
self.network_middleware = STRANGER_CONFIGURATION
if network_middleware:
raise self.ConfigurationError(
"Cannot configure a stranger to use network middleware.")
#
# Learner
#
self.learn_on_same_thread = learn_on_same_thread
self.abort_on_learning_error = abort_on_learning_error
self.start_learning_now = start_learning_now
self.save_metadata = save_metadata
self.reload_metadata = reload_metadata
self.__fleet_state = FleetStateTracker()
known_nodes = known_nodes or set()
if known_nodes:
self.known_nodes._nodes.update(
{node.checksum_public_address: node
for node in known_nodes})
self.known_nodes.record_fleet_state(
) # TODO: Does this call need to be here?
#
# Blockchain
#
self.poa = poa
self.provider_uri = provider_uri or self.DEFAULT_PROVIDER_URI
self.blockchain = NO_BLOCKCHAIN_CONNECTION
self.accounts = NO_BLOCKCHAIN_CONNECTION
self.token_agent = NO_BLOCKCHAIN_CONNECTION
self.miner_agent = NO_BLOCKCHAIN_CONNECTION
self.policy_agent = NO_BLOCKCHAIN_CONNECTION
#
# Development Mode
#
if dev_mode:
# Ephemeral dev settings
self.abort_on_learning_error = True
self.save_metadata = False
self.reload_metadata = False
# Generate one-time alphanumeric development password
alphabet = string.ascii_letters + string.digits
password = ''.join(secrets.choice(alphabet) for _ in range(32))
# Auto-initialize
self.initialize(password=password,
import_registry=import_seed_registry)
def __call__(self, *args, **kwargs):
return self.produce(*args, **kwargs)
@classmethod
def generate(cls, password: str, no_registry: bool, *args,
**kwargs) -> 'UrsulaConfiguration':
"""Shortcut: Hook-up a new initial installation and write configuration file to the disk"""
ursula_config = cls(dev_mode=False, is_me=True, *args, **kwargs)
ursula_config.__write(password=password, no_registry=no_registry)
return ursula_config
def __write(self, password: str, no_registry: bool):
_new_installation_path = self.initialize(password=password,
import_registry=no_registry)
_configuration_filepath = self.to_configuration_file(
filepath=self.config_file_location)
def cleanup(self) -> None:
if self.__dev_mode:
self.__temp_dir.cleanup()
@property
def dev_mode(self):
return self.__dev_mode
@property
def known_nodes(self):
return self.__fleet_state
def connect_to_blockchain(self, recompile_contracts: bool = False):
if self.federated_only:
raise NodeConfiguration.ConfigurationError(
"Cannot connect to blockchain in federated mode")
self.blockchain = Blockchain.connect(provider_uri=self.provider_uri,
compile=recompile_contracts,
poa=self.poa)
self.accounts = self.blockchain.interface.w3.eth.accounts
self.log.debug("Established connection to provider {}".format(
self.blockchain.interface.provider_uri))
def connect_to_contracts(self) -> None:
"""Initialize contract agency and set them on config"""
self.token_agent = NucypherTokenAgent(blockchain=self.blockchain)
self.miner_agent = MinerAgent(blockchain=self.blockchain)
self.policy_agent = PolicyAgent(blockchain=self.blockchain)
self.log.debug("Established connection to nucypher contracts")
def read_known_nodes(self):
known_nodes = self.node_storage.all(federated_only=self.federated_only)
known_nodes = {
node.checksum_public_address: node
for node in known_nodes
}
self.known_nodes._nodes.update(known_nodes)
self.known_nodes.record_fleet_state()
return self.known_nodes
def forget_nodes(self) -> None:
self.node_storage.clear()
message = "Removed all stored node node metadata and certificates"
self.log.debug(message)
def destroy(self, force: bool = False, logs: bool = True) -> None:
# TODO: Further confirm this is a nucypher dir first! (in-depth measure)
if logs is True or force:
shutil.rmtree(USER_LOG_DIR, ignore_errors=True)
try:
shutil.rmtree(self.config_root, ignore_errors=force)
except FileNotFoundError:
raise FileNotFoundError("No such directory {}".format(
self.config_root))
def generate_parameters(self, **overrides) -> dict:
merged_parameters = {
**self.static_payload,
**self.dynamic_payload,
**overrides
}
non_init_params = ('config_root', 'poa', 'provider_uri')
character_init_params = filter(lambda t: t[0] not in non_init_params,
merged_parameters.items())
return dict(character_init_params)
def produce(self, **overrides):
"""Initialize a new character instance and return it."""
merged_parameters = self.generate_parameters(**overrides)
character = self._CHARACTER_CLASS(**merged_parameters)
return character
@staticmethod
def _read_configuration_file(filepath: str) -> dict:
try:
with open(filepath, 'r') as file:
raw_contents = file.read()
payload = NodeConfiguration.__CONFIG_FILE_DESERIALIZER(
raw_contents)
except FileNotFoundError as e:
raise # TODO: Do we need better exception handling here?
return payload
@classmethod
def from_configuration_file(cls,
filepath: str = None,
**overrides) -> 'NodeConfiguration':
"""Initialize a NodeConfiguration from a JSON file."""
from nucypher.config.storages import NodeStorage
node_storage_subclasses = {
storage._name: storage
for storage in NodeStorage.__subclasses__()
}
if filepath is None:
filepath = cls.DEFAULT_CONFIG_FILE_LOCATION
# Read from disk
payload = cls._read_configuration_file(filepath=filepath)
# Initialize NodeStorage subclass from file (sub-configuration)
storage_payload = payload['node_storage']
storage_type = storage_payload[NodeStorage._TYPE_LABEL]
storage_class = node_storage_subclasses[storage_type]
node_storage = storage_class.from_payload(
payload=storage_payload,
# character_class=cls._CHARACTER_CLASS, # TODO: Do not pass this here - Always Use Ursula
federated_only=payload['federated_only'],
serializer=cls.NODE_SERIALIZER,
deserializer=cls.NODE_DESERIALIZER)
# Deserialize domains to UTF-8 bytestrings
domains = list(domain.encode() for domain in payload['domains'])
payload.update(dict(node_storage=node_storage, domains=domains))
# Filter out Nones from overrides to detect, well, overrides
overrides = {k: v for k, v in overrides.items() if v is not None}
# Instantiate from merged params
node_configuration = cls(**{**payload, **overrides})
return node_configuration
def to_configuration_file(self, filepath: str = None) -> str:
"""Write the static_payload to a JSON file."""
if filepath is None:
filename = '{}{}'.format(self._NAME.lower(),
self.__CONFIG_FILE_EXT)
filepath = os.path.join(self.config_root, filename)
payload = self.static_payload
del payload['is_me'] # TODO
# Serialize domains
domains = list(str(d) for d in self.domains)
# Save node connection data
payload.update(
dict(node_storage=self.node_storage.payload(), domains=domains))
with open(filepath, 'w') as config_file:
config_file.write(json.dumps(payload, indent=4))
return filepath
def validate(self, config_root: str, no_registry=False) -> bool:
# Top-level
if not os.path.exists(config_root):
raise self.ConfigurationError(
'No configuration directory found at {}.'.format(config_root))
# Sub-paths
filepaths = self.runtime_filepaths
if no_registry:
del filepaths['registry_filepath']
for field, path in filepaths.items():
if not os.path.exists(path):
message = 'Missing configuration file or directory: {}.'
if 'registry' in path:
message += ' Did you mean to pass --federated-only?'
raise NodeConfiguration.InvalidConfiguration(
message.format(path))
return True
@property
def static_payload(self) -> dict:
"""Exported static configuration values for initializing Ursula"""
payload = dict(
config_root=self.config_root,
# Identity
is_me=self.is_me,
federated_only=self.federated_only,
checksum_public_address=self.checksum_public_address,
keyring_dir=self.keyring_dir,
# Behavior
domains=self.domains, # From Set
learn_on_same_thread=self.learn_on_same_thread,
abort_on_learning_error=self.abort_on_learning_error,
start_learning_now=self.start_learning_now,
save_metadata=self.save_metadata,
)
if not self.federated_only:
payload.update(dict(provider_uri=self.provider_uri, poa=self.poa))
return payload
@property
def dynamic_payload(self, **overrides) -> dict:
"""Exported dynamic configuration values for initializing Ursula"""
if self.reload_metadata:
known_nodes = self.node_storage.all(
federated_only=self.federated_only)
known_nodes = {
node.checksum_public_address: node
for node in known_nodes
}
self.known_nodes._nodes.update(known_nodes)
self.known_nodes.record_fleet_state()
payload = dict(network_middleware=self.network_middleware
or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS(),
known_nodes=self.known_nodes,
node_storage=self.node_storage,
crypto_power_ups=self.derive_node_power_ups() or None)
if not self.federated_only:
self.connect_to_blockchain(recompile_contracts=False)
payload.update(blockchain=self.blockchain)
if overrides:
self.log.debug(
"Overrides supplied to dynamic payload for {}".format(
self.__class__.__name__))
payload.update(overrides)
return payload
@property
def runtime_filepaths(self):
filepaths = dict(config_root=self.config_root,
keyring_dir=self.keyring_dir,
registry_filepath=self.registry_filepath)
return filepaths
@classmethod
def generate_runtime_filepaths(cls, config_root: str) -> dict:
"""Dynamically generate paths based on configuration root directory"""
filepaths = dict(
config_root=config_root,
config_file_location=os.path.join(config_root,
cls.CONFIG_FILENAME),
keyring_dir=os.path.join(config_root, 'keyring'),
registry_filepath=os.path.join(config_root,
NodeConfiguration.__REGISTRY_NAME))
return filepaths
def _cache_runtime_filepaths(self) -> None:
"""Generate runtime filepaths and cache them on the config object"""
filepaths = self.generate_runtime_filepaths(
config_root=self.config_root)
for field, filepath in filepaths.items():
if getattr(self, field) is UNINITIALIZED_CONFIGURATION:
setattr(self, field, filepath)
def derive_node_power_ups(self) -> List[CryptoPowerUp]:
power_ups = list()
if self.is_me and not self.dev_mode:
for power_class in self._CHARACTER_CLASS._default_crypto_powerups:
power_up = self.keyring.derive_crypto_power(power_class)
power_ups.append(power_up)
return power_ups
def initialize(
self,
password: str,
import_registry: bool = True,
) -> str:
"""Initialize a new configuration."""
#
# Create Config Root
#
if self.__dev_mode:
self.__temp_dir = TemporaryDirectory(
prefix=self.TEMP_CONFIGURATION_DIR_PREFIX)
self.config_root = self.__temp_dir.name
else:
try:
os.mkdir(self.config_root, mode=0o755)
except FileExistsError:
if os.listdir(self.config_root):
message = "There are existing files located at {}".format(
self.config_root)
raise self.ConfigurationError(message)
except FileNotFoundError:
os.makedirs(self.config_root, mode=0o755)
#
# Create Config Subdirectories
#
self._cache_runtime_filepaths()
try:
# Node Storage
self.node_storage.initialize()
# Keyring
if not self.dev_mode:
os.mkdir(
self.keyring_dir,
mode=0o700) # keyring TODO: Keyring backend entry point
self.write_keyring(password=password)
# Registry
if import_registry and not self.federated_only:
self.write_registry(
output_filepath=self.registry_filepath, # type: str
source=self.__registry_source, # type: str
blank=import_registry) # type: bool
except FileExistsError:
existing_paths = [
os.path.join(self.config_root, f)
for f in os.listdir(self.config_root)
]
message = "There are pre-existing files at {}: {}".format(
self.config_root, existing_paths)
self.log.critical(message)
raise NodeConfiguration.ConfigurationError(message)
if not self.__dev_mode:
self.validate(config_root=self.config_root,
no_registry=import_registry or self.federated_only)
# Success
message = "Created nucypher installation files at {}".format(
self.config_root)
self.log.debug(message)
return self.config_root
def attach_keyring(self,
checksum_address: str = None,
*args,
**kwargs) -> None:
if self.keyring is not NO_KEYRING_ATTACHED:
if self.keyring.checksum_address != (checksum_address or
self.checksum_public_address):
raise self.ConfigurationError(
"There is already a keyring attached to this configuration."
)
return
if (checksum_address or self.checksum_public_address) is None:
raise self.ConfigurationError(
"No account specified to unlock keyring")
self.keyring = NucypherKeyring(
keyring_root=self.keyring_dir, # type: str
account=checksum_address
or self.checksum_public_address, # type: str
*args,
**kwargs)
def write_keyring(self, password: str,
**generation_kwargs) -> NucypherKeyring:
self.keyring = NucypherKeyring.generate(password=password,
keyring_root=self.keyring_dir,
**generation_kwargs)
# Operating mode switch TODO: #466
if self.federated_only:
self.checksum_public_address = self.keyring.federated_address
else:
self.checksum_public_address = self.keyring.checksum_address
return self.keyring
def write_registry(self,
output_filepath: str = None,
source: str = None,
force: bool = False,
blank=False) -> str:
if force and os.path.isfile(output_filepath):
raise self.ConfigurationError(
'There is an existing file at the registry output_filepath {}'.
format(output_filepath))
output_filepath = output_filepath or self.registry_filepath
source = source or self.REGISTRY_SOURCE
if not blank and not self.dev_mode:
# Validate Registry
with open(source, 'r') as registry_file:
try:
json.loads(registry_file.read())
except JSONDecodeError:
message = "The registry source {} is not valid JSON".format(
source)
self.log.critical(message)
raise self.ConfigurationError(message)
else:
self.log.debug(
"Source registry {} is valid JSON".format(source))
else:
self.log.warn("Writing blank registry")
open(output_filepath, 'w').close() # write blank
self.log.debug(
"Successfully wrote registry to {}".format(output_filepath))
return output_filepath
def make_rest_app(
db_filepath: str,
this_node,
serving_domains,
log=Logger("http-application-layer")
) -> Tuple:
forgetful_node_storage = ForgetfulNodeStorage(federated_only=this_node.federated_only)
from nucypher.keystore import keystore
from nucypher.keystore.db import Base
from sqlalchemy.engine import create_engine
log.info("Starting datastore {}".format(db_filepath))
# See: https://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#connect-strings
if db_filepath:
db_uri = f'sqlite:///{db_filepath}'
else:
db_uri = 'sqlite://' # TODO: Is this a sane default? See #667
engine = create_engine(db_uri)
Base.metadata.create_all(engine)
datastore = keystore.KeyStore(engine)
db_engine = engine
from nucypher.characters.lawful import Alice, Ursula
_alice_class = Alice
_node_class = Ursula
rest_app = Flask("ursula-service")
@rest_app.route("/public_information")
def public_information():
"""
REST endpoint for public keys and address.
"""
response = Response(
response=bytes(this_node),
mimetype='application/octet-stream')
return response
@rest_app.route('/node_metadata', methods=["GET"])
def all_known_nodes():
headers = {'Content-Type': 'application/octet-stream'}
if this_node.known_nodes.checksum is NO_KNOWN_NODES:
return Response(b"", headers=headers, status=204)
known_nodes_bytestring = this_node.bytestring_of_known_nodes()
signature = this_node.stamp(known_nodes_bytestring)
return Response(bytes(signature) + known_nodes_bytestring, headers=headers)
@rest_app.route('/node_metadata', methods=["POST"])
def node_metadata_exchange():
# If these nodes already have the same fleet state, no exchange is necessary.
learner_fleet_state = request.args.get('fleet')
if learner_fleet_state == this_node.known_nodes.checksum:
log.debug("Learner already knew fleet state {}; doing nothing.".format(learner_fleet_state))
headers = {'Content-Type': 'application/octet-stream'}
payload = this_node.known_nodes.snapshot() + bytes(FLEET_STATES_MATCH)
signature = this_node.stamp(payload)
return Response(bytes(signature) + payload, headers=headers)
sprouts = _node_class.batch_from_bytes(request.data,
registry=this_node.registry)
# TODO: This logic is basically repeated in learn_from_teacher_node and remember_node.
# Let's find a better way. #555
for node in sprouts:
@crosstown_traffic()
def learn_about_announced_nodes():
if node in this_node.known_nodes:
if node.timestamp <= this_node.known_nodes[node.checksum_address].timestamp:
return
node.mature()
try:
node.verify_node(this_node.network_middleware.client,
registry=this_node.registry,
)
# Suspicion
except node.SuspiciousActivity as e:
# TODO: Include data about caller?
# TODO: Account for possibility that stamp, rather than interface, was bad.
# TODO: Maybe also record the bytes representation separately to disk?
message = f"Suspicious Activity about {node}: {str(e)}. Announced via REST."
log.warn(message)
this_node.suspicious_activities_witnessed['vladimirs'].append(node)
except NodeSeemsToBeDown as e:
# This is a rather odd situation - this node *just* contacted us and asked to be verified. Where'd it go? Maybe a NAT problem?
log.info(f"Node announced itself to us just now, but seems to be down: {node}. Response was {e}.")
log.debug(f"Phantom node certificate: {node.certificate}")
# Async Sentinel
except Exception as e:
log.critical(f"This exception really needs to be handled differently: {e}")
raise
# Believable
else:
log.info("Learned about previously unknown node: {}".format(node))
this_node.remember_node(node)
# TODO: Record new fleet state
# Cleanup
finally:
forgetful_node_storage.forget()
# TODO: What's the right status code here? 202? Different if we already knew about the node?
return all_known_nodes()
@rest_app.route('/consider_arrangement', methods=['POST'])
def consider_arrangement():
from nucypher.policy.policies import Arrangement
arrangement = Arrangement.from_bytes(request.data)
with ThreadedSession(db_engine) as session:
new_policy_arrangement = datastore.add_policy_arrangement(
arrangement.expiration.datetime(),
id=arrangement.id.hex().encode(),
alice_verifying_key=arrangement.alice.stamp,
session=session,
)
# TODO: Make the rest of this logic actually work - do something here
# to decide if this Arrangement is worth accepting.
headers = {'Content-Type': 'application/octet-stream'}
# TODO: Make this a legit response #234.
return Response(b"This will eventually be an actual acceptance of the arrangement.", headers=headers)
@rest_app.route("/kFrag/<id_as_hex>", methods=['POST'])
def set_policy(id_as_hex):
"""
REST endpoint for setting a kFrag.
TODO: Instead of taking a Request, use the apistar typing system to type
a payload and validate / split it.
TODO: Validate that the kfrag being saved is pursuant to an approved
Policy (see #121).
"""
policy_message_kit = UmbralMessageKit.from_bytes(request.data)
alices_verifying_key = policy_message_kit.sender_verifying_key
alice = _alice_class.from_public_keys(verifying_key=alices_verifying_key)
try:
cleartext = this_node.verify_from(alice, policy_message_kit, decrypt=True)
except InvalidSignature:
# TODO: Perhaps we log this?
return Response(status_code=400)
kfrag = KFrag.from_bytes(cleartext)
if not kfrag.verify(signing_pubkey=alices_verifying_key):
raise InvalidSignature("{} is invalid".format(kfrag))
with ThreadedSession(db_engine) as session:
datastore.attach_kfrag_to_saved_arrangement(
alice,
id_as_hex,
kfrag,
session=session)
# TODO: Sign the arrangement here. #495
return "" # TODO: Return A 200, with whatever policy metadata.
@rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"])
def revoke_arrangement(id_as_hex):
"""
REST endpoint for revoking/deleting a KFrag from a node.
"""
from nucypher.policy.collections import Revocation
revocation = Revocation.from_bytes(request.data)
log.info("Received revocation: {} -- for arrangement {}".format(bytes(revocation).hex(), id_as_hex))
try:
with ThreadedSession(db_engine) as session:
# Verify the Notice was signed by Alice
policy_arrangement = datastore.get_policy_arrangement(
id_as_hex.encode(), session=session)
alice_pubkey = UmbralPublicKey.from_bytes(
policy_arrangement.alice_verifying_key.key_data)
# Check that the request is the same for the provided revocation
if id_as_hex != revocation.arrangement_id.hex():
log.debug("Couldn't identify an arrangement with id {}".format(id_as_hex))
return Response(status_code=400)
elif revocation.verify_signature(alice_pubkey):
datastore.del_policy_arrangement(
id_as_hex.encode(), session=session)
except (NotFound, InvalidSignature) as e:
log.debug("Exception attempting to revoke: {}".format(e))
return Response(response='KFrag not found or revocation signature is invalid.', status=404)
else:
log.info("KFrag successfully removed.")
return Response(response='KFrag deleted!', status=200)
@rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"])
def reencrypt_via_rest(id_as_hex):
# Get Policy Arrangement
try:
arrangement_id = binascii.unhexlify(id_as_hex)
except (binascii.Error, TypeError):
return Response(response=b'Invalid arrangement ID', status=405)
try:
with ThreadedSession(db_engine) as session:
arrangement = datastore.get_policy_arrangement(arrangement_id=id_as_hex.encode(), session=session)
except NotFound:
return Response(response=arrangement_id, status=404)
# Get KFrag
kfrag = KFrag.from_bytes(arrangement.kfrag)
# Get Work Order
from nucypher.policy.collections import WorkOrder # Avoid circular import
alice_verifying_key_bytes = arrangement.alice_verifying_key.key_data
alice_verifying_key = UmbralPublicKey.from_bytes(alice_verifying_key_bytes)
alice_address = canonical_address_from_umbral_key(alice_verifying_key)
work_order_payload = request.data
work_order = WorkOrder.from_rest_payload(arrangement_id=arrangement_id,
rest_payload=work_order_payload,
ursula=this_node,
alice_address=alice_address)
log.info(f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}")
# Re-encrypt
response = this_node._reencrypt(kfrag=kfrag,
work_order=work_order,
alice_verifying_key=alice_verifying_key)
# Now, Ursula saves this workorder to her database...
with ThreadedSession(db_engine):
this_node.datastore.save_workorder(bob_verifying_key=bytes(work_order.bob.stamp),
bob_signature=bytes(work_order.receipt_signature),
arrangement_id=work_order.arrangement_id)
headers = {'Content-Type': 'application/octet-stream'}
return Response(headers=headers, response=response)
@rest_app.route('/treasure_map/<treasure_map_id>')
def provide_treasure_map(treasure_map_id):
headers = {'Content-Type': 'application/octet-stream'}
treasure_map_index = bytes.fromhex(treasure_map_id)
try:
treasure_map = this_node.treasure_maps[treasure_map_index]
response = Response(bytes(treasure_map), headers=headers)
log.info("{} providing TreasureMap {}".format(this_node.nickname, treasure_map_id))
except KeyError:
log.info("{} doesn't have requested TreasureMap {}".format(this_node.stamp, treasure_map_id))
response = Response("No Treasure Map with ID {}".format(treasure_map_id),
status=404, headers=headers)
return response
@rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST'])
def receive_treasure_map(treasure_map_id):
from nucypher.policy.collections import TreasureMap
try:
treasure_map = TreasureMap.from_bytes(bytes_representation=request.data, verify=True)
except TreasureMap.InvalidSignature:
do_store = False
else:
do_store = treasure_map.public_id() == treasure_map_id
if do_store:
log.info("{} storing TreasureMap {}".format(this_node, treasure_map_id))
# TODO 341 - what if we already have this TreasureMap?
treasure_map_index = bytes.fromhex(treasure_map_id)
this_node.treasure_maps[treasure_map_index] = treasure_map
return Response(bytes(treasure_map), status=202)
else:
# TODO: Make this a proper 500 or whatever.
log.info("Bad TreasureMap ID; not storing {}".format(treasure_map_id))
assert False
@rest_app.route('/status/', methods=['GET'])
def status():
if request.args.get('json'):
payload = this_node.abridged_node_details()
response = jsonify(payload)
return response
else:
headers = {"Content-Type": "text/html", "charset": "utf-8"}
previous_states = list(reversed(this_node.known_nodes.states.values()))[:5]
# Mature every known node before rendering.
for node in this_node.known_nodes:
node.mature()
try:
content = status_template.render(this_node=this_node,
known_nodes=this_node.known_nodes,
previous_states=previous_states,
domains=serving_domains,
version=nucypher.__version__,
checksum_address=this_node.checksum_address)
except Exception as e:
log.debug("Template Rendering Exception: ".format(str(e)))
raise TemplateError(str(e)) from e
return Response(response=content, headers=headers)
return rest_app, datastore
def _make_rest_app(datastore: Datastore, this_node, serving_domain: str, log: Logger) -> Tuple[Flask, Datastore]:
forgetful_node_storage = ForgetfulNodeStorage(federated_only=this_node.federated_only) # FIXME: Seems unused
from nucypher.characters.lawful import Alice, Ursula
_alice_class = Alice
_node_class = Ursula
rest_app = Flask("ursula-service")
rest_app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD_CONTENT_LENGTH
@rest_app.route("/public_information")
def public_information():
"""
REST endpoint for public keys and address.
"""
response = Response(
response=bytes(this_node),
mimetype='application/octet-stream')
return response
@rest_app.route("/ping", methods=['POST'])
def ping():
"""
Asks this node: "Can you access my public information endpoint"?
"""
try:
requesting_ursula = Ursula.from_bytes(request.data, registry=this_node.registry)
requesting_ursula.mature()
except ValueError: # (ValueError)
return Response({'error': 'Invalid Ursula'}, status=400)
else:
initiator_address, initiator_port = tuple(requesting_ursula.rest_interface)
# Compare requester and posted Ursula information
request_address = request.environ['REMOTE_ADDR']
if request_address != initiator_address:
return Response({'error': 'Suspicious origin address'}, status=400)
#
# Make a Sandwich
#
try:
# Fetch and store initiator's teacher certificate.
certificate = this_node.network_middleware.get_certificate(host=initiator_address, port=initiator_port)
certificate_filepath = this_node.node_storage.store_node_certificate(certificate=certificate)
requesting_ursula_bytes = this_node.network_middleware.client.node_information(host=initiator_address,
port=initiator_port,
certificate_filepath=certificate_filepath)
except NodeSeemsToBeDown:
return Response({'error': 'Unreachable node'}, status=400) # ... toasted
except InvalidNodeCertificate:
return Response({'error': 'Invalid TLS certificate - missing checksum address'}, status=400) # ... invalid
# Compare the results of the outer POST with the inner GET... yum
if requesting_ursula_bytes == request.data:
return Response(status=200)
else:
return Response({'error': 'Suspicious node'}, status=400)
@rest_app.route('/node_metadata', methods=["GET"])
def all_known_nodes():
headers = {'Content-Type': 'application/octet-stream'}
if this_node.known_nodes.checksum is NO_KNOWN_NODES:
return Response(b"", headers=headers, status=204)
known_nodes_bytestring = this_node.bytestring_of_known_nodes()
signature = this_node.stamp(known_nodes_bytestring)
return Response(bytes(signature) + known_nodes_bytestring, headers=headers)
@rest_app.route('/node_metadata', methods=["POST"])
def node_metadata_exchange():
# If these nodes already have the same fleet state, no exchange is necessary.
learner_fleet_state = request.args.get('fleet')
if learner_fleet_state == this_node.known_nodes.checksum:
log.debug("Learner already knew fleet state {}; doing nothing.".format(learner_fleet_state))
headers = {'Content-Type': 'application/octet-stream'}
payload = this_node.known_nodes.snapshot() + bytes(FLEET_STATES_MATCH)
signature = this_node.stamp(payload)
return Response(bytes(signature) + payload, headers=headers)
sprouts = _node_class.batch_from_bytes(request.data)
for node in sprouts:
this_node.remember_node(node)
# TODO: What's the right status code here? 202? Different if we already knew about the node(s)?
return all_known_nodes()
@rest_app.route('/consider_arrangement', methods=['POST'])
def consider_arrangement():
from nucypher.policy.policies import Arrangement
arrangement = Arrangement.from_bytes(request.data)
# TODO: Look at the expiration and figure out if we're even staking that long. 1701
with datastore.describe(PolicyArrangement, arrangement.id.hex(), writeable=True) as new_policy_arrangement:
new_policy_arrangement.arrangement_id = arrangement.id.hex().encode()
new_policy_arrangement.expiration = arrangement.expiration
new_policy_arrangement.alice_verifying_key = arrangement.alice.stamp.as_umbral_pubkey()
# TODO: Fine, we'll add the arrangement here, but if we never hear from Alice again to enact it,
# we need to prune it at some point. #1700
headers = {'Content-Type': 'application/octet-stream'}
# TODO: Make this a legit response #234.
return Response(b"This will eventually be an actual acceptance of the arrangement.", headers=headers)
@rest_app.route("/kFrag/<id_as_hex>", methods=['POST'])
def set_policy(id_as_hex):
"""
REST endpoint for setting a kFrag.
"""
policy_message_kit = UmbralMessageKit.from_bytes(request.data)
alices_verifying_key = policy_message_kit.sender_verifying_key
alice = _alice_class.from_public_keys(verifying_key=alices_verifying_key)
try:
cleartext = this_node.verify_from(alice, policy_message_kit, decrypt=True)
except InvalidSignature:
# TODO: Perhaps we log this? Essentially 355.
return Response(status_code=400)
if not this_node.federated_only:
# This splitter probably belongs somewhere canonical.
transaction_splitter = BytestringSplitter(32)
tx, kfrag_bytes = transaction_splitter(cleartext, return_remainder=True)
try:
# Get all of the arrangements and verify that we'll be paid.
# TODO: We'd love for this part to be impossible to reduce the risk of collusion. #1274
arranged_addresses = this_node.policy_agent.fetch_arrangement_addresses_from_policy_txid(tx, timeout=this_node.synchronous_query_timeout)
except TimeExhausted:
# Alice didn't pay. Return response with that weird status code.
this_node.suspicious_activities_witnessed['freeriders'].append((alice, f"No transaction matching {tx}."))
return Response(status=402)
this_node_has_been_arranged = this_node.checksum_address in arranged_addresses
if not this_node_has_been_arranged:
this_node.suspicious_activities_witnessed['freeriders'].append((alice, f"The transaction {tx} does not list me as a Worker - it lists {arranged_addresses}."))
return Response(status=402)
else:
_tx = NO_BLOCKCHAIN_CONNECTION
kfrag_bytes = cleartext
kfrag = KFrag.from_bytes(kfrag_bytes)
if not kfrag.verify(signing_pubkey=alices_verifying_key):
raise InvalidSignature("{} is invalid".format(kfrag))
with datastore.describe(PolicyArrangement, id_as_hex, writeable=True) as policy_arrangement:
if not policy_arrangement.alice_verifying_key == alice.stamp.as_umbral_pubkey():
raise alice.SuspiciousActivity
policy_arrangement.kfrag = kfrag
# TODO: Sign the arrangement here. #495
return "" # TODO: Return A 200, with whatever policy metadata.
@rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"])
def revoke_arrangement(id_as_hex):
"""
REST endpoint for revoking/deleting a KFrag from a node.
"""
from nucypher.policy.collections import Revocation
revocation = Revocation.from_bytes(request.data)
log.info("Received revocation: {} -- for arrangement {}".format(bytes(revocation).hex(), id_as_hex))
# Check that the request is the same for the provided revocation
if not id_as_hex == revocation.arrangement_id.hex():
log.debug("Couldn't identify an arrangement with id {}".format(id_as_hex))
return Response(status_code=400)
try:
with datastore.describe(PolicyArrangement, id_as_hex, writeable=True) as policy_arrangement:
if revocation.verify_signature(policy_arrangement.alice_verifying_key):
policy_arrangement.delete()
except (DatastoreTransactionError, InvalidSignature) as e:
log.debug("Exception attempting to revoke: {}".format(e))
return Response(response='KFrag not found or revocation signature is invalid.', status=404)
else:
log.info("KFrag successfully removed.")
return Response(response='KFrag deleted!', status=200)
@rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"])
def reencrypt_via_rest(id_as_hex):
# Get Policy Arrangement
try:
arrangement_id = binascii.unhexlify(id_as_hex)
except (binascii.Error, TypeError):
return Response(response=b'Invalid arrangement ID', status=405)
try:
# Get KFrag
# TODO: Yeah, well, what if this arrangement hasn't been enacted? 1702
with datastore.describe(PolicyArrangement, id_as_hex) as policy_arrangement:
kfrag = policy_arrangement.kfrag
alice_verifying_key = policy_arrangement.alice_verifying_key
except RecordNotFound:
return Response(response=arrangement_id, status=404)
# Get Work Order
from nucypher.policy.collections import WorkOrder # Avoid circular import
alice_address = canonical_address_from_umbral_key(alice_verifying_key)
work_order_payload = request.data
work_order = WorkOrder.from_rest_payload(arrangement_id=arrangement_id,
rest_payload=work_order_payload,
ursula=this_node,
alice_address=alice_address)
log.info(f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}")
# Re-encrypt
response = this_node._reencrypt(kfrag=kfrag,
work_order=work_order,
alice_verifying_key=alice_verifying_key)
# Now, Ursula saves this workorder to her database...
# Note: we give the work order a random ID to store it under.
with datastore.describe(Workorder, str(uuid.uuid4()), writeable=True) as new_workorder:
new_workorder.arrangement_id = work_order.arrangement_id
new_workorder.bob_verifying_key = work_order.bob.stamp.as_umbral_pubkey()
new_workorder.bob_signature = work_order.receipt_signature
headers = {'Content-Type': 'application/octet-stream'}
return Response(headers=headers, response=response)
@rest_app.route('/treasure_map/<treasure_map_id>')
def provide_treasure_map(treasure_map_id):
headers = {'Content-Type': 'application/octet-stream'}
treasure_map_index = bytes.fromhex(treasure_map_id)
try:
treasure_map = this_node.treasure_maps[treasure_map_index]
response = Response(bytes(treasure_map), headers=headers)
log.info("{} providing TreasureMap {}".format(this_node.nickname, treasure_map_id))
except KeyError:
log.info("{} doesn't have requested TreasureMap {}".format(this_node.stamp, treasure_map_id))
response = Response("No Treasure Map with ID {}".format(treasure_map_id),
status=404, headers=headers)
return response
@rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST'])
def receive_treasure_map(treasure_map_id):
# TODO: Any of the codepaths that trigger 4xx Responses here are also SuspiciousActivity.
if not this_node.federated_only:
from nucypher.policy.collections import SignedTreasureMap as _MapClass
else:
from nucypher.policy.collections import TreasureMap as _MapClass
try:
treasure_map = _MapClass.from_bytes(bytes_representation=request.data, verify=True)
except _MapClass.InvalidSignature:
log.info("Bad TreasureMap HRAC Signature; not storing {}".format(treasure_map_id))
return Response("This TreasureMap's HRAC is not properly signed.", status=401)
treasure_map_index = bytes.fromhex(treasure_map_id)
# First let's see if we already have this map.
try:
previously_saved_map = this_node.treasure_maps[treasure_map_index]
except KeyError:
pass # We don't have the map. We'll validate and perhaps save it.
else:
if previously_saved_map == treasure_map:
return Response("Already have this map.", status=303)
# Otherwise, if it's a different map with the same ID, we move on to validation.
if treasure_map.public_id() == treasure_map_id:
do_store = True
else:
return Response("Can't save a TreasureMap with this ID from you.", status=409)
if do_store and not this_node.federated_only:
alice_checksum_address = this_node.policy_agent.contract.functions.getPolicyOwner(
treasure_map._hrac[:16]).call()
do_store = treasure_map.verify_blockchain_signature(checksum_address=alice_checksum_address)
if do_store:
log.info("{} storing TreasureMap {}".format(this_node, treasure_map_id))
this_node.treasure_maps[treasure_map_index] = treasure_map
return Response(bytes(treasure_map), status=202)
else:
log.info("Bad TreasureMap ID; not storing {}".format(treasure_map_id))
return Response("This TreasureMap doesn't match a paid Policy.", status=402)
@rest_app.route('/status/', methods=['GET'])
def status():
if request.args.get('json'):
payload = this_node.abridged_node_details()
response = jsonify(payload)
return response
else:
headers = {"Content-Type": "text/html", "charset": "utf-8"}
previous_states = list(reversed(this_node.known_nodes.states.values()))[:5]
# Mature every known node before rendering.
for node in this_node.known_nodes:
node.mature()
try:
content = status_template.render(this_node=this_node,
known_nodes=this_node.known_nodes,
previous_states=previous_states,
domain=serving_domain,
version=nucypher.__version__,
checksum_address=this_node.checksum_address)
except Exception as e:
log.debug("Template Rendering Exception: ".format(str(e)))
raise TemplateError(str(e)) from e
return Response(response=content, headers=headers)
return rest_app
class TestInMemoryNodeStorage(BaseTestNodeStorageBackends):
storage_backend = ForgetfulNodeStorage(
character_class=BaseTestNodeStorageBackends.character_class,
federated_only=BaseTestNodeStorageBackends.federated_only)
storage_backend.initialize()
def make_rest_app(
db_filepath: str,
network_middleware: RestMiddleware,
federated_only: bool,
treasure_map_tracker: dict,
node_tracker: 'FleetStateTracker',
node_bytes_caster: Callable,
work_order_tracker: list,
node_nickname: str,
node_recorder: Callable,
stamp: SignatureStamp,
verifier: Callable,
suspicious_activity_tracker: dict,
serving_domains,
log=Logger("http-application-layer")) -> Tuple:
forgetful_node_storage = ForgetfulNodeStorage(
federated_only=federated_only)
from nucypher.keystore import keystore
from nucypher.keystore.db import Base
from sqlalchemy.engine import create_engine
log.info("Starting datastore {}".format(db_filepath))
# See: https://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#connect-strings
if db_filepath:
db_uri = f'sqlite:///{db_filepath}'
else:
db_uri = 'sqlite://' # TODO: Is this a sane default? See #667
engine = create_engine(db_uri)
Base.metadata.create_all(engine)
datastore = keystore.KeyStore(engine)
db_engine = engine
from nucypher.characters.lawful import Alice, Ursula
_alice_class = Alice
_node_class = Ursula
rest_app = Flask("ursula-service")
@rest_app.route("/public_information")
def public_information():
"""
REST endpoint for public keys and address..
"""
response = Response(response=node_bytes_caster(),
mimetype='application/octet-stream')
return response
@rest_app.route('/node_metadata', methods=["GET"])
def all_known_nodes():
headers = {'Content-Type': 'application/octet-stream'}
if node_tracker.checksum is NO_KNOWN_NODES:
return Response(b"", headers=headers, status=204)
payload = node_tracker.snapshot()
ursulas_as_vbytes = (VariableLengthBytestring(n) for n in node_tracker)
ursulas_as_bytes = bytes().join(bytes(u) for u in ursulas_as_vbytes)
ursulas_as_bytes += VariableLengthBytestring(node_bytes_caster())
payload += ursulas_as_bytes
signature = stamp(payload)
return Response(bytes(signature) + payload, headers=headers)
@rest_app.route('/node_metadata', methods=["POST"])
def node_metadata_exchange():
# If these nodes already have the same fleet state, no exchange is necessary.
learner_fleet_state = request.args.get('fleet')
if learner_fleet_state == node_tracker.checksum:
log.debug(
"Learner already knew fleet state {}; doing nothing.".format(
learner_fleet_state))
headers = {'Content-Type': 'application/octet-stream'}
payload = node_tracker.snapshot() + bytes(FLEET_STATES_MATCH)
signature = stamp(payload)
return Response(bytes(signature) + payload, headers=headers)
nodes = _node_class.batch_from_bytes(
request.data, federated_only=federated_only) # TODO: 466
# TODO: This logic is basically repeated in learn_from_teacher_node and remember_node.
# Let's find a better way. #555
for node in nodes:
if GLOBAL_DOMAIN not in serving_domains:
if not serving_domains.intersection(node.serving_domains):
continue # This node is not serving any of our domains.
if node in node_tracker:
if node.timestamp <= node_tracker[
node.checksum_public_address].timestamp:
continue
@crosstown_traffic()
def learn_about_announced_nodes():
try:
certificate_filepath = forgetful_node_storage.store_node_certificate(
certificate=node.certificate)
node.verify_node(
network_middleware,
accept_federated_only=federated_only, # TODO: 466
certificate_filepath=certificate_filepath)
# Suspicion
except node.SuspiciousActivity:
# TODO: Include data about caller?
# TODO: Account for possibility that stamp, rather than interface, was bad.
# TODO: Maybe also record the bytes representation separately to disk?
message = "Suspicious Activity: Discovered node with bad signature: {}. Announced via REST."
log.warn(message)
suspicious_activity_tracker['vladimirs'].append(node)
# Async Sentinel
except Exception as e:
log.critical(str(e))
raise
# Believable
else:
log.info(
"Learned about previously unknown node: {}".format(
node))
node_recorder(node)
# TODO: Record new fleet state
# Cleanup
finally:
forgetful_node_storage.forget()
# TODO: What's the right status code here? 202? Different if we already knew about the node?
return all_known_nodes()
@rest_app.route('/consider_arrangement', methods=['POST'])
def consider_arrangement():
from nucypher.policy.models import Arrangement
arrangement = Arrangement.from_bytes(request.data)
with ThreadedSession(db_engine) as session:
new_policy_arrangement = datastore.add_policy_arrangement(
arrangement.expiration.datetime(),
id=arrangement.id.hex().encode(),
alice_pubkey_sig=arrangement.alice.stamp,
session=session,
)
# TODO: Make the rest of this logic actually work - do something here
# to decide if this Arrangement is worth accepting.
headers = {'Content-Type': 'application/octet-stream'}
# TODO: Make this a legit response #234.
return Response(
b"This will eventually be an actual acceptance of the arrangement.",
headers=headers)
@rest_app.route("/kFrag/<id_as_hex>", methods=['POST'])
def set_policy(id_as_hex):
"""
REST endpoint for setting a kFrag.
TODO: Instead of taking a Request, use the apistar typing system to type
a payload and validate / split it.
TODO: Validate that the kfrag being saved is pursuant to an approved
Policy (see #121).
"""
policy_message_kit = UmbralMessageKit.from_bytes(request.data)
alices_verifying_key = policy_message_kit.sender_pubkey_sig
alice = _alice_class.from_public_keys(
{SigningPower: alices_verifying_key})
try:
cleartext = verifier(alice, policy_message_kit, decrypt=True)
except InvalidSignature:
# TODO: Perhaps we log this?
return Response(status_code=400)
kfrag = KFrag.from_bytes(cleartext)
if not kfrag.verify(signing_pubkey=alices_verifying_key):
raise InvalidSignature("{} is invalid".format(kfrag))
with ThreadedSession(db_engine) as session:
datastore.attach_kfrag_to_saved_arrangement(alice,
id_as_hex,
kfrag,
session=session)
# TODO: Sign the arrangement here. #495
return "" # TODO: Return A 200, with whatever policy metadata.
@rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"])
def revoke_arrangement(id_as_hex):
"""
REST endpoint for revoking/deleting a KFrag from a node.
"""
from nucypher.policy.models import Revocation
revocation = Revocation.from_bytes(request.data)
log.info("Received revocation: {} -- for arrangement {}".format(
bytes(revocation).hex(), id_as_hex))
try:
with ThreadedSession(db_engine) as session:
# Verify the Notice was signed by Alice
policy_arrangement = datastore.get_policy_arrangement(
id_as_hex.encode(), session=session)
alice_pubkey = UmbralPublicKey.from_bytes(
policy_arrangement.alice_pubkey_sig.key_data)
# Check that the request is the same for the provided revocation
if id_as_hex != revocation.arrangement_id.hex():
log.debug(
"Couldn't identify an arrangement with id {}".format(
id_as_hex))
return Response(status_code=400)
elif revocation.verify_signature(alice_pubkey):
datastore.del_policy_arrangement(id_as_hex.encode(),
session=session)
except (NotFound, InvalidSignature) as e:
log.debug("Exception attempting to revoke: {}".format(e))
return Response(
response='KFrag not found or revocation signature is invalid.',
status=404)
else:
log.info("KFrag successfully removed.")
return Response(response='KFrag deleted!', status=200)
@rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"])
def reencrypt_via_rest(id_as_hex):
from nucypher.policy.models import WorkOrder # Avoid circular import
arrangement_id = binascii.unhexlify(id_as_hex)
work_order = WorkOrder.from_rest_payload(arrangement_id, request.data)
log.info("Work Order from {}, signed {}".format(
work_order.bob, work_order.receipt_signature))
with ThreadedSession(db_engine) as session:
policy_arrangement = datastore.get_policy_arrangement(
arrangement_id=id_as_hex.encode(), session=session)
kfrag_bytes = policy_arrangement.kfrag # Careful! :-)
verifying_key_bytes = policy_arrangement.alice_pubkey_sig.key_data
# TODO: Push this to a lower level. Perhaps to Ursula character? #619
kfrag = KFrag.from_bytes(kfrag_bytes)
alices_verifying_key = UmbralPublicKey.from_bytes(verifying_key_bytes)
cfrag_byte_stream = b""
alices_address = canonical_address_from_umbral_key(
alices_verifying_key)
if not alices_address == work_order.alice_address:
message = f"This Bob ({work_order.bob}) sent an Alice's ETH address " \
f"({work_order.alice_address}) that doesn't match " \
f"the one I have ({alices_address})."
raise SuspiciousActivity(message)
bob_pubkey = work_order.bob.stamp.as_umbral_pubkey()
if not work_order.alice_address_signature.verify(
message=alices_address, verifying_key=bob_pubkey):
message = f"This Bob ({work_order.bob}) sent an invalid signature of Alice's ETH address"
raise InvalidSignature(message)
# This is Bob's signature of Alice's verifying key as ETH address.
alice_address_signature = bytes(work_order.alice_address_signature)
for capsule, capsule_signature in zip(work_order.capsules,
work_order.capsule_signatures):
# This is the capsule signed by Bob
capsule_signature = bytes(capsule_signature)
# Ursula signs on top of it. Now both are committed to the same capsule.
# She signs Alice's address too.
ursula_signature = stamp(capsule_signature +
alice_address_signature)
capsule.set_correctness_keys(verifying=alices_verifying_key)
cfrag = pre.reencrypt(kfrag,
capsule,
metadata=bytes(ursula_signature))
log.info(f"Re-encrypting for {capsule}, made {cfrag}.")
signature = stamp(bytes(cfrag) + bytes(capsule))
cfrag_byte_stream += VariableLengthBytestring(cfrag) + signature
# TODO: Put this in Ursula's datastore
work_order_tracker.append(work_order)
headers = {'Content-Type': 'application/octet-stream'}
return Response(response=cfrag_byte_stream, headers=headers)
@rest_app.route('/treasure_map/<treasure_map_id>')
def provide_treasure_map(treasure_map_id):
headers = {'Content-Type': 'application/octet-stream'}
treasure_map_bytes = keccak_digest(binascii.unhexlify(treasure_map_id))
try:
treasure_map = treasure_map_tracker[treasure_map_bytes]
response = Response(bytes(treasure_map), headers=headers)
log.info("{} providing TreasureMap {}".format(
node_nickname, treasure_map_id))
except KeyError:
log.info("{} doesn't have requested TreasureMap {}".format(
stamp, treasure_map_id))
response = Response(
"No Treasure Map with ID {}".format(treasure_map_id),
status=404,
headers=headers)
return response
@rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST'])
def receive_treasure_map(treasure_map_id):
from nucypher.policy.models import TreasureMap
try:
treasure_map = TreasureMap.from_bytes(
bytes_representation=request.data, verify=True)
except TreasureMap.InvalidSignature:
do_store = False
else:
do_store = treasure_map.public_id() == treasure_map_id
if do_store:
log.info("{} storing TreasureMap {}".format(
stamp, treasure_map_id))
# # # #
# TODO: Now that the DHT is retired, let's do this another way.
# self.dht_server.set_now(binascii.unhexlify(treasure_map_id),
# constants.BYTESTRING_IS_TREASURE_MAP + bytes(treasure_map))
# # # #
# TODO 341 - what if we already have this TreasureMap?
treasure_map_tracker[keccak_digest(
binascii.unhexlify(treasure_map_id))] = treasure_map
return Response(bytes(treasure_map), status=202)
else:
# TODO: Make this a proper 500 or whatever.
log.info(
"Bad TreasureMap ID; not storing {}".format(treasure_map_id))
assert False
@rest_app.route('/status')
def status():
# TODO: Seems very strange to deserialize *this node* when we can just pass it in.
# Might be a sign that we need to rethnk this composition.
headers = {"Content-Type": "text/html", "charset": "utf-8"}
this_node = _node_class.from_bytes(node_bytes_caster(),
federated_only=federated_only)
previous_states = list(reversed(node_tracker.states.values()))[:5]
try:
content = status_template.render(this_node=this_node,
known_nodes=node_tracker,
previous_states=previous_states)
except Exception as e:
log.debug("Template Rendering Exception: ".format(str(e)))
raise TemplateError(str(e)) from e
return Response(response=content, headers=headers)
return rest_app, datastore
def from_seed_and_stake_info(
cls,
seed_uri: str,
federated_only: bool,
minimum_stake: int = 0,
checksum_address: str = None, # TODO: Why is this unused?
network_middleware: RestMiddleware = None,
*args,
**kwargs) -> 'Ursula':
#
# WARNING: xxx Poison xxx
# Let's learn what we can about the ... "seednode".
#
if network_middleware is None:
network_middleware = RestMiddleware()
host, port, checksum_address = parse_node_uri(seed_uri)
# Fetch the hosts TLS certificate and read the common name
certificate = network_middleware.get_certificate(host=host, port=port)
real_host = certificate.subject.get_attributes_for_oid(
NameOID.COMMON_NAME)[0].value
temp_node_storage = ForgetfulNodeStorage(federated_only=federated_only)
certificate_filepath = temp_node_storage.store_node_certificate(
certificate=certificate)
# Load the host as a potential seed node
potential_seed_node = cls.from_rest_url(
host=real_host,
port=port,
network_middleware=network_middleware,
certificate_filepath=certificate_filepath,
federated_only=True,
*args,
**kwargs) # TODO: 466
potential_seed_node.certificate_filepath = certificate_filepath
if checksum_address:
# Ensure this is the specific node we expected
if not checksum_address == potential_seed_node.checksum_public_address:
template = "This seed node has a different wallet address: {} (expected {}). Are you sure this is a seednode?"
raise potential_seed_node.SuspiciousActivity(
template.format(
potential_seed_node.checksum_public_address,
checksum_address))
# Check the node's stake (optional)
if minimum_stake > 0:
# TODO: check the blockchain to verify that address has more then minimum_stake. #511
raise NotImplementedError("Stake checking is not implemented yet.")
# Verify the node's TLS certificate
try:
potential_seed_node.verify_node(
network_middleware=network_middleware,
accept_federated_only=federated_only,
certificate_filepath=certificate_filepath)
except potential_seed_node.InvalidNode:
raise # TODO: What if our seed node fails verification?
# OK - everyone get out
temp_node_storage.forget()
return potential_seed_node
def make_rest_app(
db_filepath: str,
this_node,
serving_domains,
log=Logger("http-application-layer")
) -> Tuple:
forgetful_node_storage = ForgetfulNodeStorage(federated_only=this_node.federated_only)
from nucypher.datastore import datastore
from nucypher.datastore.db import Base
from sqlalchemy.engine import create_engine
log.info("Starting datastore {}".format(db_filepath))
# See: https://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#connect-strings
if db_filepath:
db_uri = f'sqlite:///{db_filepath}'
else:
db_uri = 'sqlite://' # TODO: Is this a sane default? See #667
engine = create_engine(db_uri)
Base.metadata.create_all(engine)
datastore = datastore.Datastore(engine)
db_engine = engine
from nucypher.characters.lawful import Alice, Ursula
_alice_class = Alice
_node_class = Ursula
rest_app = Flask("ursula-service")
rest_app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD_CONTENT_LENGTH
@rest_app.route("/public_information")
def public_information():
"""
REST endpoint for public keys and address.
"""
response = Response(
response=bytes(this_node),
mimetype='application/octet-stream')
return response
@rest_app.route("/ping", methods=['POST'])
def ping():
"""
Asks this node: "Can you access my public information endpoint"?
"""
try:
requesting_ursula = Ursula.from_bytes(request.data, registry=this_node.registry)
requesting_ursula.mature()
except ValueError: # (ValueError)
return Response({'error': 'Invalid Ursula'}, status=400)
else:
initiator_address, initiator_port = tuple(requesting_ursula.rest_interface)
# Compare requester and posted Ursula information
request_address = request.environ['REMOTE_ADDR']
if request_address != initiator_address:
return Response({'error': 'Suspicious origin address'}, status=400)
#
# Make a Sandwich
#
try:
# Fetch and store initiator's teacher certificate.
certificate = this_node.network_middleware.get_certificate(host=initiator_address, port=initiator_port)
certificate_filepath = this_node.node_storage.store_node_certificate(certificate=certificate)
requesting_ursula_bytes = this_node.network_middleware.client.node_information(host=initiator_address,
port=initiator_port,
certificate_filepath=certificate_filepath)
except NodeSeemsToBeDown:
return Response({'error': 'Unreachable node'}, status=400) # ... toasted
# Compare the results of the outer POST with the inner GET... yum
if requesting_ursula_bytes == request.data:
return Response(status=200)
else:
return Response({'error': 'Suspicious node'}, status=400)
@rest_app.route('/node_metadata', methods=["GET"])
def all_known_nodes():
headers = {'Content-Type': 'application/octet-stream'}
if this_node.known_nodes.checksum is NO_KNOWN_NODES:
return Response(b"", headers=headers, status=204)
known_nodes_bytestring = this_node.bytestring_of_known_nodes()
signature = this_node.stamp(known_nodes_bytestring)
return Response(bytes(signature) + known_nodes_bytestring, headers=headers)
@rest_app.route('/node_metadata', methods=["POST"])
def node_metadata_exchange():
# If these nodes already have the same fleet state, no exchange is necessary.
learner_fleet_state = request.args.get('fleet')
if learner_fleet_state == this_node.known_nodes.checksum:
log.debug("Learner already knew fleet state {}; doing nothing.".format(learner_fleet_state))
headers = {'Content-Type': 'application/octet-stream'}
payload = this_node.known_nodes.snapshot() + bytes(FLEET_STATES_MATCH)
signature = this_node.stamp(payload)
return Response(bytes(signature) + payload, headers=headers)
sprouts = _node_class.batch_from_bytes(request.data,
registry=this_node.registry)
# TODO: This logic is basically repeated in learn_from_teacher_node and remember_node.
# Let's find a better way. #555
for node in sprouts:
@crosstown_traffic()
def learn_about_announced_nodes():
if node in this_node.known_nodes:
if node.timestamp <= this_node.known_nodes[node.checksum_address].timestamp:
return
node.mature()
try:
node.verify_node(this_node.network_middleware.client,
registry=this_node.registry)
# Suspicion
except node.SuspiciousActivity as e:
# 355
# TODO: Include data about caller?
# TODO: Account for possibility that stamp, rather than interface, was bad.
# TODO: Maybe also record the bytes representation separately to disk?
message = f"Suspicious Activity about {node}: {str(e)}. Announced via REST."
log.warn(message)
this_node.suspicious_activities_witnessed['vladimirs'].append(node)
except NodeSeemsToBeDown as e:
# This is a rather odd situation - this node *just* contacted us and asked to be verified. Where'd it go? Maybe a NAT problem?
log.info(f"Node announced itself to us just now, but seems to be down: {node}. Response was {e}.")
log.debug(f"Phantom node certificate: {node.certificate}")
# Async Sentinel
except Exception as e:
log.critical(f"This exception really needs to be handled differently: {e}")
raise
# Believable
else:
log.info("Learned about previously unknown node: {}".format(node))
this_node.remember_node(node)
# TODO: Record new fleet state
# Cleanup
finally:
forgetful_node_storage.forget()
# TODO: What's the right status code here? 202? Different if we already knew about the node?
return all_known_nodes()
@rest_app.route('/consider_arrangement', methods=['POST'])
def consider_arrangement():
from nucypher.policy.policies import Arrangement
arrangement = Arrangement.from_bytes(request.data)
# TODO: Look at the expiration and figure out if we're even staking that long. 1701
with ThreadedSession(db_engine) as session:
new_policy_arrangement = datastore.add_policy_arrangement(
arrangement.expiration.datetime(),
arrangement_id=arrangement.id.hex().encode(),
alice_verifying_key=arrangement.alice.stamp,
session=session,
)
# TODO: Fine, we'll add the arrangement here, but if we never hear from Alice again to enact it,
# we need to prune it at some point. #1700
headers = {'Content-Type': 'application/octet-stream'}
# TODO: Make this a legit response #234.
return Response(b"This will eventually be an actual acceptance of the arrangement.", headers=headers)
@rest_app.route("/kFrag/<id_as_hex>", methods=['POST'])
def set_policy(id_as_hex):
"""
REST endpoint for setting a kFrag.
"""
policy_message_kit = UmbralMessageKit.from_bytes(request.data)
alices_verifying_key = policy_message_kit.sender_verifying_key
alice = _alice_class.from_public_keys(verifying_key=alices_verifying_key)
try:
cleartext = this_node.verify_from(alice, policy_message_kit, decrypt=True)
except InvalidSignature:
# TODO: Perhaps we log this? Essentially 355.
return Response(status_code=400)
if not this_node.federated_only:
# This splitter probably belongs somewhere canonical.
transaction_splitter = BytestringSplitter(32)
tx, kfrag_bytes = transaction_splitter(cleartext, return_remainder=True)
try:
# Get all of the arrangements and verify that we'll be paid.
# TODO: We'd love for this part to be impossible to reduce the risk of collusion. #1274
arranged_addresses = this_node.policy_agent.fetch_arrangement_addresses_from_policy_txid(tx, timeout=this_node.synchronous_query_timeout)
except TimeExhausted:
# Alice didn't pay. Return response with that weird status code.
this_node.suspicious_activities_witnessed['freeriders'].append((alice, f"No transaction matching {tx}."))
return Response(status=402)
this_node_has_been_arranged = this_node.checksum_address in arranged_addresses
if not this_node_has_been_arranged:
this_node.suspicious_activities_witnessed['freeriders'].append((alice, f"The transaction {tx} does not list me as a Worker - it lists {arranged_addresses}."))
return Response(status=402)
else:
_tx = NO_BLOCKCHAIN_CONNECTION
kfrag_bytes = cleartext
kfrag = KFrag.from_bytes(kfrag_bytes)
if not kfrag.verify(signing_pubkey=alices_verifying_key):
raise InvalidSignature("{} is invalid".format(kfrag))
with ThreadedSession(db_engine) as session:
datastore.attach_kfrag_to_saved_arrangement(
alice,
id_as_hex,
kfrag,
session=session)
# TODO: Sign the arrangement here. #495
return "" # TODO: Return A 200, with whatever policy metadata.
@rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"])
def revoke_arrangement(id_as_hex):
"""
REST endpoint for revoking/deleting a KFrag from a node.
"""
from nucypher.policy.collections import Revocation
revocation = Revocation.from_bytes(request.data)
log.info("Received revocation: {} -- for arrangement {}".format(bytes(revocation).hex(), id_as_hex))
try:
with ThreadedSession(db_engine) as session:
# Verify the Notice was signed by Alice
policy_arrangement = datastore.get_policy_arrangement(
id_as_hex.encode(), session=session)
alice_pubkey = UmbralPublicKey.from_bytes(
policy_arrangement.alice_verifying_key.key_data)
# Check that the request is the same for the provided revocation
if id_as_hex != revocation.arrangement_id.hex():
log.debug("Couldn't identify an arrangement with id {}".format(id_as_hex))
return Response(status_code=400)
elif revocation.verify_signature(alice_pubkey):
datastore.del_policy_arrangement(
id_as_hex.encode(), session=session)
except (NotFound, InvalidSignature) as e:
log.debug("Exception attempting to revoke: {}".format(e))
return Response(response='KFrag not found or revocation signature is invalid.', status=404)
else:
log.info("KFrag successfully removed.")
return Response(response='KFrag deleted!', status=200)
@rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"])
def reencrypt_via_rest(id_as_hex):
# Get Policy Arrangement
try:
arrangement_id = binascii.unhexlify(id_as_hex)
except (binascii.Error, TypeError):
return Response(response=b'Invalid arrangement ID', status=405)
try:
with ThreadedSession(db_engine) as session:
arrangement = datastore.get_policy_arrangement(arrangement_id=id_as_hex.encode(), session=session)
except NotFound:
return Response(response=arrangement_id, status=404)
# Get KFrag
# TODO: Yeah, well, what if this arrangement hasn't been enacted? 1702
kfrag = KFrag.from_bytes(arrangement.kfrag)
# Get Work Order
from nucypher.policy.collections import WorkOrder # Avoid circular import
alice_verifying_key_bytes = arrangement.alice_verifying_key.key_data
alice_verifying_key = UmbralPublicKey.from_bytes(alice_verifying_key_bytes)
alice_address = canonical_address_from_umbral_key(alice_verifying_key)
work_order_payload = request.data
work_order = WorkOrder.from_rest_payload(arrangement_id=arrangement_id,
rest_payload=work_order_payload,
ursula=this_node,
alice_address=alice_address)
log.info(f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}")
# Re-encrypt
response = this_node._reencrypt(kfrag=kfrag,
work_order=work_order,
alice_verifying_key=alice_verifying_key)
# Now, Ursula saves this workorder to her database...
with ThreadedSession(db_engine):
this_node.datastore.save_workorder(bob_verifying_key=bytes(work_order.bob.stamp),
bob_signature=bytes(work_order.receipt_signature),
arrangement_id=work_order.arrangement_id)
headers = {'Content-Type': 'application/octet-stream'}
return Response(headers=headers, response=response)
@rest_app.route('/treasure_map/<treasure_map_id>')
def provide_treasure_map(treasure_map_id):
headers = {'Content-Type': 'application/octet-stream'}
treasure_map_index = bytes.fromhex(treasure_map_id)
try:
treasure_map = this_node.treasure_maps[treasure_map_index]
response = Response(bytes(treasure_map), headers=headers)
log.info("{} providing TreasureMap {}".format(this_node.nickname, treasure_map_id))
except KeyError:
log.info("{} doesn't have requested TreasureMap {}".format(this_node.stamp, treasure_map_id))
response = Response("No Treasure Map with ID {}".format(treasure_map_id),
status=404, headers=headers)
return response
@rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST'])
def receive_treasure_map(treasure_map_id):
from nucypher.policy.collections import TreasureMap
try:
treasure_map = TreasureMap.from_bytes(bytes_representation=request.data, verify=True)
except TreasureMap.InvalidSignature:
do_store = False
else:
# TODO: If we include the policy ID in this check, does that prevent map spam? 1736
do_store = treasure_map.public_id() == treasure_map_id
if do_store:
log.info("{} storing TreasureMap {}".format(this_node, treasure_map_id))
# TODO 341 - what if we already have this TreasureMap?
treasure_map_index = bytes.fromhex(treasure_map_id)
this_node.treasure_maps[treasure_map_index] = treasure_map
return Response(bytes(treasure_map), status=202)
else:
# TODO: Make this a proper 500 or whatever. #341
log.info("Bad TreasureMap ID; not storing {}".format(treasure_map_id))
assert False
@rest_app.route('/status/', methods=['GET'])
def status():
if request.args.get('json'):
payload = this_node.abridged_node_details()
response = jsonify(payload)
return response
else:
headers = {"Content-Type": "text/html", "charset": "utf-8"}
previous_states = list(reversed(this_node.known_nodes.states.values()))[:5]
# Mature every known node before rendering.
for node in this_node.known_nodes:
node.mature()
try:
content = status_template.render(this_node=this_node,
known_nodes=this_node.known_nodes,
previous_states=previous_states,
domains=serving_domains,
version=nucypher.__version__,
checksum_address=this_node.checksum_address)
except Exception as e:
log.debug("Template Rendering Exception: ".format(str(e)))
raise TemplateError(str(e)) from e
return Response(response=content, headers=headers)
return rest_app, datastore
class NodeConfiguration(ABC):
"""
'Sideways Engagement' of Character classes; a reflection of input parameters.
"""
# Abstract
_NAME = NotImplemented
_CHARACTER_CLASS = NotImplemented
CONFIG_FILENAME = NotImplemented
DEFAULT_CONFIG_FILE_LOCATION = NotImplemented
# Mode
DEFAULT_OPERATING_MODE = 'decentralized'
# Domains
DEFAULT_DOMAIN = 'goerli'
# Serializers
NODE_SERIALIZER = binascii.hexlify
NODE_DESERIALIZER = binascii.unhexlify
# System
__CONFIG_FILE_EXT = '.config'
__CONFIG_FILE_DESERIALIZER = json.loads
TEMP_CONFIGURATION_DIR_PREFIX = "nucypher-tmp-"
# Blockchain
DEFAULT_PROVIDER_URI = 'http://localhost:8545'
# Registry
__REGISTRY_NAME = 'contract_registry.json'
REGISTRY_SOURCE = os.path.join(BASE_DIR, __REGISTRY_NAME)
# Rest + TLS
DEFAULT_REST_HOST = '127.0.0.1'
DEFAULT_REST_PORT = 9151
DEFAULT_DEVELOPMENT_REST_PORT = 10151
DEFAULT_CONTROLLER_PORT = NotImplemented
__DEFAULT_TLS_CURVE = ec.SECP384R1
__DEFAULT_NETWORK_MIDDLEWARE_CLASS = RestMiddleware
class ConfigurationError(RuntimeError):
pass
class InvalidConfiguration(ConfigurationError):
pass
class NoConfigurationRoot(InvalidConfiguration):
pass
def __init__(self,
# Base
config_root: str = None,
config_file_location: str = None,
# Mode
dev_mode: bool = False,
federated_only: bool = False,
# Identity
is_me: bool = True,
checksum_address: str = None,
crypto_power: CryptoPower = None,
# Keyring
keyring: NucypherKeyring = None,
keyring_dir: str = None,
# Learner
learn_on_same_thread: bool = False,
abort_on_learning_error: bool = False,
start_learning_now: bool = True,
# REST
rest_host: str = None,
rest_port: int = None,
controller_port: int = None,
# TLS
tls_curve: EllipticCurve = None,
certificate: Certificate = None,
# Network
domains: Set[str] = None,
interface_signature: Signature = None,
network_middleware: RestMiddleware = None,
# Node Storage
known_nodes: set = None,
node_storage: NodeStorage = None,
reload_metadata: bool = True,
save_metadata: bool = True,
# Blockchain
poa: bool = False,
provider_uri: str = None,
provider_process = None,
# Registry
registry_source: str = None,
registry_filepath: str = None,
download_registry: bool = True
) -> None:
# Logs
self.log = Logger(self.__class__.__name__)
#
# REST + TLS + Web
#
self.controller_port = controller_port or self.DEFAULT_CONTROLLER_PORT
self.rest_host = rest_host or self.DEFAULT_REST_HOST
default_port = (self.DEFAULT_DEVELOPMENT_REST_PORT if dev_mode else self.DEFAULT_REST_PORT)
self.rest_port = rest_port or default_port
self.tls_curve = tls_curve or self.__DEFAULT_TLS_CURVE
self.certificate = certificate
self.interface_signature = interface_signature
self.crypto_power = crypto_power
#
# Keyring
#
self.keyring = keyring or NO_KEYRING_ATTACHED
self.keyring_dir = keyring_dir or UNINITIALIZED_CONFIGURATION
# Contract Registry
self.download_registry = download_registry
self.__registry_source = registry_source or self.REGISTRY_SOURCE
self.registry_filepath = registry_filepath or UNINITIALIZED_CONFIGURATION
#
# Configuration
#
self.config_file_location = config_file_location or UNINITIALIZED_CONFIGURATION
self.config_root = UNINITIALIZED_CONFIGURATION
#
# Mode
#
self.federated_only = federated_only
self.__dev_mode = dev_mode
if self.__dev_mode:
self.__temp_dir = UNINITIALIZED_CONFIGURATION
self.node_storage = ForgetfulNodeStorage(federated_only=federated_only, character_class=self.__class__)
else:
self.__temp_dir = LIVE_CONFIGURATION
self.config_root = config_root or DEFAULT_CONFIG_ROOT
self._cache_runtime_filepaths()
self.node_storage = node_storage or LocalFileBasedNodeStorage(federated_only=federated_only,
config_root=self.config_root)
# Domains
self.domains = domains or {self.DEFAULT_DOMAIN}
#
# Identity
#
self.is_me = is_me
self.checksum_address = checksum_address
if self.is_me is True or dev_mode is True:
# Self
if self.checksum_address and dev_mode is False:
self.attach_keyring()
self.network_middleware = network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS()
else:
# Stranger
self.node_storage = STRANGER_CONFIGURATION
self.keyring_dir = STRANGER_CONFIGURATION
self.keyring = STRANGER_CONFIGURATION
self.network_middleware = STRANGER_CONFIGURATION
if network_middleware:
raise self.ConfigurationError("Cannot configure a stranger to use network middleware.")
#
# Learner
#
self.learn_on_same_thread = learn_on_same_thread
self.abort_on_learning_error = abort_on_learning_error
self.start_learning_now = start_learning_now
self.save_metadata = save_metadata
self.reload_metadata = reload_metadata
self.__fleet_state = FleetStateTracker()
known_nodes = known_nodes or set()
if known_nodes:
self.known_nodes._nodes.update({node.checksum_address: node for node in known_nodes})
self.known_nodes.record_fleet_state() # TODO: Does this call need to be here?
#
# Blockchain
#
self.poa = poa
self.provider_uri = provider_uri or self.DEFAULT_PROVIDER_URI
self.provider_process = provider_process or NO_BLOCKCHAIN_CONNECTION
self.blockchain = NO_BLOCKCHAIN_CONNECTION.bool_value(False)
self.accounts = NO_BLOCKCHAIN_CONNECTION
self.token_agent = NO_BLOCKCHAIN_CONNECTION
self.miner_agent = NO_BLOCKCHAIN_CONNECTION
self.policy_agent = NO_BLOCKCHAIN_CONNECTION
#
# Development Mode
#
if dev_mode:
# Ephemeral dev settings
self.abort_on_learning_error = True
self.save_metadata = False
self.reload_metadata = False
# Generate one-time alphanumeric development password
alphabet = string.ascii_letters + string.digits
password = ''.join(secrets.choice(alphabet) for _ in range(32))
# Auto-initialize
self.initialize(password=password, download_registry=download_registry)
def __call__(self, *args, **kwargs):
return self.produce(*args, **kwargs)
@classmethod
def generate(cls, password: str, *args, **kwargs):
"""Shortcut: Hook-up a new initial installation and write configuration file to the disk"""
node_config = cls(dev_mode=False, is_me=True, *args, **kwargs)
node_config.__write(password=password)
return node_config
def __write(self, password: str):
_new_installation_path = self.initialize(password=password, download_registry=self.download_registry)
_configuration_filepath = self.to_configuration_file(filepath=self.config_file_location)
def cleanup(self) -> None:
if self.__dev_mode:
self.__temp_dir.cleanup()
if self.blockchain:
self.blockchain.disconnect()
@property
def dev_mode(self):
return self.__dev_mode
@property
def known_nodes(self):
return self.__fleet_state
def connect_to_blockchain(self,
enode: str = None,
recompile_contracts: bool = False,
full_sync: bool = False) -> None:
"""
:param enode: ETH seednode or bootnode enode address to start learning from,
i.e. 'enode://[email protected]:30303'
:param recompile_contracts: Recompile all contracts on connection.
:return: None
"""
if self.federated_only:
raise NodeConfiguration.ConfigurationError("Cannot connect to blockchain in federated mode")
self.blockchain = Blockchain.connect(provider_uri=self.provider_uri,
compile=recompile_contracts,
poa=self.poa,
fetch_registry=True,
provider_process=self.provider_process,
sync=full_sync)
# Read Ethereum Node Keyring
self.accounts = self.blockchain.interface.w3.eth.accounts
# Add Ethereum Peer
if enode:
if self.blockchain.interface.client_version == 'geth':
self.blockchain.interface.w3.geth.admin.addPeer(enode)
else:
raise NotImplementedError
def connect_to_contracts(self) -> None:
"""Initialize contract agency and set them on config"""
self.token_agent = NucypherTokenAgent(blockchain=self.blockchain)
self.miner_agent = MinerAgent(blockchain=self.blockchain)
self.policy_agent = PolicyAgent(blockchain=self.blockchain)
self.log.debug("Established connection to nucypher contracts")
def read_known_nodes(self):
known_nodes = self.node_storage.all(federated_only=self.federated_only)
known_nodes = {node.checksum_address: node for node in known_nodes}
self.known_nodes._nodes.update(known_nodes)
self.known_nodes.record_fleet_state()
return self.known_nodes
def forget_nodes(self) -> None:
self.node_storage.clear()
message = "Removed all stored node node metadata and certificates"
self.log.debug(message)
def destroy(self) -> None:
"""Parse a node configuration and remove all associated files from the filesystem"""
self.keyring.destroy()
os.remove(self.config_file_location)
def generate_parameters(self, **overrides) -> dict:
merged_parameters = {**self.static_payload, **self.dynamic_payload, **overrides}
non_init_params = ('config_root', 'poa', 'provider_uri')
character_init_params = filter(lambda t: t[0] not in non_init_params, merged_parameters.items())
return dict(character_init_params)
def produce(self, **overrides):
"""Initialize a new character instance and return it."""
merged_parameters = self.generate_parameters(**overrides)
character = self._CHARACTER_CLASS(**merged_parameters)
return character
@staticmethod
def _read_configuration_file(filepath: str) -> dict:
try:
with open(filepath, 'r') as file:
raw_contents = file.read()
payload = NodeConfiguration.__CONFIG_FILE_DESERIALIZER(raw_contents)
except FileNotFoundError:
raise
return payload
@classmethod
def get_configuration_payload(cls, filepath: str = None, **overrides) -> dict:
from nucypher.config.storages import NodeStorage
node_storage_subclasses = {storage._name: storage for storage in NodeStorage.__subclasses__()}
if filepath is None:
filepath = cls.DEFAULT_CONFIG_FILE_LOCATION
# Read from disk
payload = cls._read_configuration_file(filepath=filepath)
# Sanity check
try:
checksum_address = payload['checksum_address']
except KeyError:
raise cls.ConfigurationError(f"No checksum address specified in configuration file {filepath}")
else:
if not eth_utils.is_checksum_address(checksum_address):
raise cls.ConfigurationError(f"Address: '{checksum_address}', specified in {filepath} is not a valid checksum address.")
# Initialize NodeStorage subclass from file (sub-configuration)
storage_payload = payload['node_storage']
storage_type = storage_payload[NodeStorage._TYPE_LABEL]
storage_class = node_storage_subclasses[storage_type]
node_storage = storage_class.from_payload(payload=storage_payload,
federated_only=payload['federated_only'],
serializer=cls.NODE_SERIALIZER,
deserializer=cls.NODE_DESERIALIZER)
domains = set(payload['domains'])
payload.update(dict(node_storage=node_storage, domains=domains))
# Filter out Nones from overrides to detect, well, overrides
overrides = {k: v for k, v in overrides.items() if v is not None}
payload = {**payload, **overrides}
return payload
@classmethod
def from_configuration_file(cls,
filepath: str = None,
provider_process=None,
**overrides) -> 'NodeConfiguration':
"""Initialize a NodeConfiguration from a JSON file."""
payload = cls.get_configuration_payload(filepath=filepath, **overrides)
# Instantiate from merged params
node_configuration = cls(config_file_location=filepath,
provider_process=provider_process,
**payload)
return node_configuration
def to_configuration_file(self, filepath: str = None) -> str:
"""Write the static_payload to a JSON file."""
if not filepath:
filepath = os.path.join(self.config_root, self.CONFIG_FILENAME)
if os.path.isfile(filepath):
# Avoid overriding an existing default configuration
filename = f'{self._NAME.lower()}-{self.checksum_address[:6]}{self.__CONFIG_FILE_EXT}'
filepath = os.path.join(self.config_root, filename)
payload = self.static_payload
del payload['is_me']
# Save node connection data
payload.update(dict(node_storage=self.node_storage.payload(), domains=list(self.domains)))
with open(filepath, 'w') as config_file:
config_file.write(json.dumps(payload, indent=4))
return filepath
def validate(self, config_root: str, no_registry=False) -> bool:
# Top-level
if not os.path.exists(config_root):
raise self.ConfigurationError('No configuration directory found at {}.'.format(config_root))
# Sub-paths
filepaths = self.runtime_filepaths
if no_registry:
del filepaths['registry_filepath']
for field, path in filepaths.items():
if not os.path.exists(path):
message = 'Missing configuration file or directory: {}.'
if 'registry' in path:
message += ' Did you mean to pass --federated-only?'
raise NodeConfiguration.InvalidConfiguration(message.format(path))
return True
@property
def static_payload(self) -> dict:
"""Exported static configuration values for initializing Ursula"""
payload = dict(
config_root=self.config_root,
# Identity
is_me=self.is_me,
federated_only=self.federated_only,
checksum_address=self.checksum_address,
keyring_dir=self.keyring_dir,
# Behavior
domains=self.domains, # From Set
provider_uri=self.provider_uri,
learn_on_same_thread=self.learn_on_same_thread,
abort_on_learning_error=self.abort_on_learning_error,
start_learning_now=self.start_learning_now,
save_metadata=self.save_metadata,
)
if not self.federated_only:
payload.update(dict(provider_uri=self.provider_uri, poa=self.poa))
return payload
@property
def dynamic_payload(self, connect_to_blockchain: bool = True, **overrides) -> dict:
"""Exported dynamic configuration values for initializing Ursula"""
if self.reload_metadata:
known_nodes = self.node_storage.all(federated_only=self.federated_only)
known_nodes = {node.checksum_address: node for node in known_nodes}
self.known_nodes._nodes.update(known_nodes)
self.known_nodes.record_fleet_state()
payload = dict(network_middleware=self.network_middleware or self.__DEFAULT_NETWORK_MIDDLEWARE_CLASS(),
known_nodes=self.known_nodes,
node_storage=self.node_storage,
crypto_power_ups=self.derive_node_power_ups() or None)
if not self.federated_only and connect_to_blockchain:
self.connect_to_blockchain(recompile_contracts=False)
payload.update(blockchain=self.blockchain)
if overrides:
self.log.debug("Overrides supplied to dynamic payload for {}".format(self.__class__.__name__))
payload.update(overrides)
return payload
@property
def runtime_filepaths(self):
filepaths = dict(config_root=self.config_root,
keyring_dir=self.keyring_dir,
registry_filepath=self.registry_filepath)
return filepaths
@classmethod
def generate_runtime_filepaths(cls, config_root: str) -> dict:
"""Dynamically generate paths based on configuration root directory"""
filepaths = dict(config_root=config_root,
config_file_location=os.path.join(config_root, cls.CONFIG_FILENAME),
keyring_dir=os.path.join(config_root, 'keyring'),
registry_filepath=os.path.join(config_root, NodeConfiguration.__REGISTRY_NAME))
return filepaths
def _cache_runtime_filepaths(self) -> None:
"""Generate runtime filepaths and cache them on the config object"""
filepaths = self.generate_runtime_filepaths(config_root=self.config_root)
for field, filepath in filepaths.items():
if getattr(self, field) is UNINITIALIZED_CONFIGURATION:
setattr(self, field, filepath)
def derive_node_power_ups(self) -> List[CryptoPowerUp]:
power_ups = list()
if self.is_me and not self.dev_mode:
for power_class in self._CHARACTER_CLASS._default_crypto_powerups:
power_up = self.keyring.derive_crypto_power(power_class)
power_ups.append(power_up)
return power_ups
def initialize(self, password: str, download_registry: bool = True) -> str:
"""Initialize a new configuration and write installation files to disk."""
#
# Create Base System Filepaths
#
if self.__dev_mode:
self.__temp_dir = TemporaryDirectory(prefix=self.TEMP_CONFIGURATION_DIR_PREFIX)
self.config_root = self.__temp_dir.name
else:
# Production Configuration
try:
os.mkdir(self.config_root, mode=0o755)
except FileExistsError:
if os.listdir(self.config_root):
message = "There are existing files located at {}".format(self.config_root)
self.log.debug(message)
except FileNotFoundError:
os.makedirs(self.config_root, mode=0o755)
# Generate Installation Subdirectories
self._cache_runtime_filepaths()
#
# Node Storage
#
self.node_storage.initialize()
#
# Keyring
#
if not self.dev_mode:
if not os.path.isdir(self.keyring_dir):
os.mkdir(self.keyring_dir, mode=0o700) # TODO: Keyring backend entry point - COS
self.write_keyring(password=password)
#
# Registry
#
if download_registry and not self.federated_only:
self.registry_filepath = EthereumContractRegistry.download_latest_publication()
#
# Verify
#
if not self.__dev_mode:
self.validate(config_root=self.config_root, no_registry=(not download_registry) or self.federated_only)
#
# Success
#
message = "Created nucypher installation files at {}".format(self.config_root)
self.log.debug(message)
return self.config_root
def attach_keyring(self, checksum_address: str = None, *args, **kwargs) -> None:
if self.keyring is not NO_KEYRING_ATTACHED:
if self.keyring.checksum_address != (checksum_address or self.checksum_address):
raise self.ConfigurationError("There is already a keyring attached to this configuration.")
return
if (checksum_address or self.checksum_address) is None:
raise self.ConfigurationError("No account specified to unlock keyring")
self.keyring = NucypherKeyring(keyring_root=self.keyring_dir, # type: str
account=checksum_address or self.checksum_address, # type: str
*args, **kwargs)
def write_keyring(self, password: str, wallet: bool = True, **generation_kwargs) -> NucypherKeyring:
checksum_address = None
#
# Decentralized
#
if wallet:
# Note: It is assumed the blockchain is not yet available.
if not self.federated_only and not self.checksum_address:
# "Casual Geth"
if self.provider_process:
if not os.path.exists(self.provider_process.data_dir):
os.mkdir(self.provider_process.data_dir)
# Get or create wallet address (geth etherbase)
checksum_address = self.provider_process.ensure_account_exists(password=password)
# "Formal Geth" - Manual Web3 Provider, We assume is already running and available
else:
self.connect_to_blockchain()
if not self.blockchain.interface.client.accounts:
raise self.ConfigurationError(f'Web3 provider "{self.provider_uri}" does not have any accounts')
checksum_address = self.blockchain.interface.client.etherbase
# Addresses read from some node keyrings (clients) are *not* returned in checksum format.
checksum_address = to_checksum_address(checksum_address)
# Use explicit address
elif self.checksum_address:
checksum_address = self.checksum_address
self.keyring = NucypherKeyring.generate(password=password,
keyring_root=self.keyring_dir,
checksum_address=checksum_address,
**generation_kwargs)
# Operating mode switch
if self.federated_only or not wallet:
self.checksum_address = self.keyring.federated_address
else:
self.checksum_address = self.keyring.account
return self.keyring
def write_registry(self,
output_filepath: str = None,
source: str = None,
force: bool = False,
blank=False) -> str:
if force and os.path.isfile(output_filepath):
raise self.ConfigurationError(
'There is an existing file at the registry output_filepath {}'.format(output_filepath))
output_filepath = output_filepath or self.registry_filepath
source = source or self.REGISTRY_SOURCE
if not blank and not self.dev_mode:
# Validate Registry
with open(source, 'r') as registry_file:
try:
json.loads(registry_file.read())
except JSONDecodeError:
message = "The registry source {} is not valid JSON".format(source)
self.log.critical(message)
raise self.ConfigurationError(message)
else:
self.log.debug("Source registry {} is valid JSON".format(source))
else:
self.log.warn("Writing blank registry")
open(output_filepath, 'w').close() # write blank
self.log.debug("Successfully wrote registry to {}".format(output_filepath))
return output_filepath
