def get_external_ip_from_centralized_source(log: Logger = IP_DETECTION_LOGGER) -> Union[str, None]:
"""Use hardcoded URL to determine the external IP address of this host."""
ip = _request(url=CENTRALIZED_IP_ORACLE_URL)
if ip:
log.info(f'Fetched external IP address ({ip}) from centralized source ({CENTRALIZED_IP_ORACLE_URL}).')
return ip
class Porter(Learner):
BANNER = r"""
______
(_____ \ _
_____) )__ ____| |_ ____ ____
| ____/ _ \ / ___) _)/ _ )/ ___)
| | | |_| | | | |_( (/ /| |
|_| \___/|_| \___)____)_|
the Pipe for nucypher network operations
"""
APP_NAME = "Porter"
_SHORT_LEARNING_DELAY = 2
_LONG_LEARNING_DELAY = 30
_ROUNDS_WITHOUT_NODES_AFTER_WHICH_TO_SLOW_DOWN = 25
DEFAULT_EXECUTION_TIMEOUT = 10 # 10s
DEFAULT_PORT = 9155
_interface_class = PorterInterface
class UrsulaInfo(NamedTuple):
"""Simple object that stores relevant Ursula information resulting from sampling."""
checksum_address: ChecksumAddress
uri: str
encrypting_key: PublicKey
def __init__(self,
domain: str = None,
registry: BaseContractRegistry = None,
controller: bool = True,
federated_only: bool = False,
node_class: object = Ursula,
provider_uri: str = None,
*args,
**kwargs):
self.federated_only = federated_only
if not self.federated_only:
if not provider_uri:
raise ValueError(
'Provider URI is required for decentralized Porter.')
if not BlockchainInterfaceFactory.is_interface_initialized(
provider_uri=provider_uri):
BlockchainInterfaceFactory.initialize_interface(
provider_uri=provider_uri)
self.registry = registry or InMemoryContractRegistry.from_latest_publication(
network=domain)
self.staking_agent = ContractAgency.get_agent(
StakingEscrowAgent, registry=self.registry)
else:
self.registry = NO_BLOCKCHAIN_CONNECTION.bool_value(False)
node_class.set_federated_mode(federated_only)
super().__init__(save_metadata=True,
domain=domain,
node_class=node_class,
*args,
**kwargs)
self.log = Logger(self.__class__.__name__)
# Controller Interface
self.interface = self._interface_class(porter=self)
self.controller = NO_CONTROL_PROTOCOL
if controller:
# TODO need to understand this better - only made it analogous to what was done for characters
self.make_cli_controller()
self.log.info(self.BANNER)
def get_ursulas(
self,
quantity: int,
duration_periods: int = None, # optional for federated mode
exclude_ursulas: Optional[Sequence[ChecksumAddress]] = None,
include_ursulas: Optional[Sequence[ChecksumAddress]] = None
) -> List[UrsulaInfo]:
reservoir = self._make_staker_reservoir(quantity, duration_periods,
exclude_ursulas,
include_ursulas)
value_factory = PrefetchStrategy(reservoir, quantity)
def get_ursula_info(ursula_address) -> Porter.UrsulaInfo:
if ursula_address not in self.known_nodes:
raise ValueError(f"{ursula_address} is not known")
ursula = self.known_nodes[ursula_address]
try:
# verify node is valid
self.network_middleware.client.verify_and_parse_node_or_host_and_port(
node_or_sprout=ursula, host=None, port=None)
return Porter.UrsulaInfo(
checksum_address=ursula_address,
uri=f"{ursula.rest_interface.formal_uri}",
encrypting_key=ursula.public_keys(DecryptingPower))
except Exception as e:
self.log.debug(
f"Unable to obtain Ursula information ({ursula_address}): {str(e)}"
)
raise
self.block_until_number_of_known_nodes_is(
quantity,
timeout=self.DEFAULT_EXECUTION_TIMEOUT,
learn_on_this_thread=True,
eager=True)
worker_pool = WorkerPool(worker=get_ursula_info,
value_factory=value_factory,
target_successes=quantity,
timeout=self.DEFAULT_EXECUTION_TIMEOUT,
stagger_timeout=1,
threadpool_size=quantity)
worker_pool.start()
successes = worker_pool.block_until_target_successes()
ursulas_info = successes.values()
return list(ursulas_info)
def retrieve_cfrags(
self,
treasure_map: TreasureMap,
retrieval_kits: Sequence[RetrievalKit],
alice_verifying_key: PublicKey,
bob_encrypting_key: PublicKey,
bob_verifying_key: PublicKey,
) -> List[RetrievalResult]:
client = RetrievalClient(self)
return client.retrieve_cfrags(treasure_map, retrieval_kits,
alice_verifying_key, bob_encrypting_key,
bob_verifying_key)
def _make_staker_reservoir(
self,
quantity: int,
duration_periods: int = None, # optional for federated mode
exclude_ursulas: Optional[Sequence[ChecksumAddress]] = None,
include_ursulas: Optional[Sequence[ChecksumAddress]] = None):
if self.federated_only:
sample_size = quantity - (len(include_ursulas)
if include_ursulas else 0)
if not self.block_until_number_of_known_nodes_is(
sample_size,
timeout=self.DEFAULT_EXECUTION_TIMEOUT,
learn_on_this_thread=True):
raise ValueError("Unable to learn about sufficient Ursulas")
return make_federated_staker_reservoir(
known_nodes=self.known_nodes,
exclude_addresses=exclude_ursulas,
include_addresses=include_ursulas)
else:
if not duration_periods:
raise ValueError(
"Duration periods must be provided in decentralized mode")
return make_decentralized_staker_reservoir(
staking_agent=self.staking_agent,
duration_periods=duration_periods,
exclude_addresses=exclude_ursulas,
include_addresses=include_ursulas)
def make_cli_controller(self, crash_on_error: bool = False):
controller = PorterCLIController(app_name=self.APP_NAME,
crash_on_error=crash_on_error,
interface=self.interface)
self.controller = controller
return controller
def make_rpc_controller(self, crash_on_error: bool = False):
controller = JSONRPCController(app_name=self.APP_NAME,
crash_on_error=crash_on_error,
interface=self.interface)
self.controller = controller
return controller
def make_web_controller(self,
crash_on_error: bool = False,
htpasswd_filepath: Path = None,
cors_allow_origins_list: List[str] = None):
controller = WebController(
app_name=self.APP_NAME,
crash_on_error=crash_on_error,
interface=self._interface_class(porter=self))
self.controller = controller
# Register Flask Decorator
porter_flask_control = controller.make_control_transport()
# CORS origins
if cors_allow_origins_list:
try:
from flask_cors import CORS
except ImportError:
raise ImportError(
'Porter installation is required for to specify CORS origins '
'- run "pip install nucypher[porter]" and try again.')
_ = CORS(app=porter_flask_control, origins=cors_allow_origins_list)
# Basic Auth
if htpasswd_filepath:
try:
from flask_htpasswd import HtPasswdAuth
except ImportError:
raise ImportError(
'Porter installation is required for basic authentication '
'- run "pip install nucypher[porter]" and try again.')
porter_flask_control.config['FLASK_HTPASSWD_PATH'] = str(
htpasswd_filepath.absolute())
# ensure basic auth required for all endpoints
porter_flask_control.config['FLASK_AUTH_ALL'] = True
_ = HtPasswdAuth(app=porter_flask_control)
#
# Porter Control HTTP Endpoints
#
@porter_flask_control.route('/get_ursulas', methods=['GET'])
def get_ursulas() -> Response:
"""Porter control endpoint for sampling Ursulas on behalf of Alice."""
response = controller(method_name='get_ursulas',
control_request=request)
return response
@porter_flask_control.route("/revoke", methods=['POST'])
def revoke():
"""Porter control endpoint for off-chain revocation of a policy on behalf of Alice."""
response = controller(method_name='revoke',
control_request=request)
return response
@porter_flask_control.route("/retrieve_cfrags", methods=['POST'])
def retrieve_cfrags() -> Response:
"""Porter control endpoint for executing a PRE work order on behalf of Bob."""
response = controller(method_name='retrieve_cfrags',
control_request=request)
return response
return controller
class BlockchainInterface:
"""
Interacts with a solidity compiler and a registry in order to instantiate compiled
ethereum contracts with the given web3 provider backend.
"""
TIMEOUT = 600 # seconds # TODO: Correlate with the gas strategy - #2070
DEFAULT_GAS_STRATEGY = 'fast'
GAS_STRATEGIES = WEB3_GAS_STRATEGIES
Web3 = Web3 # TODO: This is name-shadowing the actual Web3. Is this intentional?
_CONTRACT_FACTORY = VersionedContract
class InterfaceError(Exception):
pass
class NoProvider(InterfaceError):
pass
class UnsupportedProvider(InterfaceError):
pass
class ConnectionFailed(InterfaceError):
pass
class UnknownContract(InterfaceError):
pass
REASONS = {
INSUFFICIENT_ETH: 'insufficient funds for gas * price + value',
}
class TransactionFailed(InterfaceError):
IPC_CODE = -32000
def __init__(self, message: str, transaction_dict: dict,
contract_function: Union[ContractFunction,
ContractConstructor], *args):
self.base_message = message
self.name = get_transaction_name(
contract_function=contract_function)
self.payload = transaction_dict
self.contract_function = contract_function
self.failures = {
BlockchainInterface.REASONS[INSUFFICIENT_ETH]:
self.insufficient_eth
}
self.message = self.failures.get(self.base_message, self.default)
super().__init__(self.message, *args)
@property
def default(self) -> str:
sender = self.payload["from"]
message = f'{self.name} from {sender[:6]}... \n' \
f'Sender balance: {prettify_eth_amount(self.get_balance())} \n' \
f'Reason: {self.base_message} \n' \
f'Transaction: {self.payload}'
return message
def get_balance(self):
blockchain = BlockchainInterfaceFactory.get_interface()
balance = blockchain.client.get_balance(
account=self.payload['from'])
return balance
@property
def insufficient_eth(self) -> str:
try:
transaction_fee = self.payload['gas'] * self.payload['gasPrice']
except KeyError:
return self.default
else:
cost = transaction_fee + self.payload.get('value', 0)
message = f'{self.name} from {self.payload["from"][:8]} - {self.base_message}.' \
f'Calculated cost is {prettify_eth_amount(cost)},' \
f'but sender only has {prettify_eth_amount(self.get_balance())}.'
return message
def __init__(
self,
emitter=None, # TODO # 1754
poa: bool = None,
light: bool = False,
provider_uri: str = NO_BLOCKCHAIN_CONNECTION,
provider: BaseProvider = NO_BLOCKCHAIN_CONNECTION,
gas_strategy: Optional[Union[str, Callable]] = None,
max_gas_price: Optional[int] = None):
"""
TODO: #1502 - Move to API docs.
Filesystem Configuration Node Client EVM
================ ====================== =============== ===================== ===========================
Solidity Files -- SolidityCompiler - --- HTTPProvider ------ ...
| |
| |
| |
- *BlockchainInterface* -- IPCProvider ----- External EVM (geth, parity...)
| |
| |
TestProvider ----- EthereumTester -------------
|
|
PyEVM (Development Chain)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Runtime Files -- --BlockchainInterface ----> Registry
| | ^
| | |
| | |
Key Files ------ CharacterConfiguration Agent ... (Contract API)
| | ^
| | |
| | |
| | Actor ...Blockchain-Character API)
| | ^
| | |
| | |
Config File --- --------- Character ... (Public API)
^
|
Human
The Blockchain is the junction of the solidity compiler, a contract registry, and a collection of
web3 network providers as a means of interfacing with the ethereum blockchain to execute
or deploy contract code on the network.
Compiler and Registry Usage
-----------------------------
Contracts are freshly re-compiled if an instance of SolidityCompiler is passed; otherwise,
The registry will read contract data saved to disk that is be used to retrieve contact address and op-codes.
Optionally, A registry instance can be passed instead.
Provider Usage
---------------
https: // github.com / ethereum / eth - tester # available-backends
* HTTP Provider - Web3 HTTP provider, typically JSON RPC 2.0 over HTTP
* Websocket Provider - Web3 WS provider, typically JSON RPC 2.0 over WS, supply endpoint uri and websocket=True
* IPC Provider - Web3 File based IPC provider transported over standard I/O
* Custom Provider - A pre-initialized web3.py provider instance to attach to this interface
"""
self.log = Logger('Blockchain')
self.poa = poa
self.provider_uri = provider_uri
self._provider = provider
self.w3 = NO_BLOCKCHAIN_CONNECTION
self.client = NO_BLOCKCHAIN_CONNECTION
self.is_light = light
# TODO: Not ready to give users total flexibility. Let's stick for the moment to known values. See #2447
if gas_strategy not in ('slow', 'medium', 'fast', 'free',
None): # FIXME: What is 'None' doing here?
raise ValueError(f"'{gas_strategy}' is an invalid gas strategy")
self.gas_strategy = gas_strategy or self.DEFAULT_GAS_STRATEGY
self.max_gas_price = max_gas_price
def __repr__(self):
r = '{name}({uri})'.format(name=self.__class__.__name__,
uri=self.provider_uri)
return r
def get_blocktime(self):
return self.client.get_blocktime()
@property
def is_connected(self) -> bool:
"""
https://web3py.readthedocs.io/en/stable/__provider.html#examples-using-automated-detection
"""
if self.client is NO_BLOCKCHAIN_CONNECTION:
return False
return self.client.is_connected
@classmethod
def get_gas_strategy(cls,
gas_strategy: Union[str,
Callable] = None) -> Callable:
try:
gas_strategy = cls.GAS_STRATEGIES[gas_strategy]
except KeyError:
if gas_strategy:
if not callable(gas_strategy):
raise ValueError(
f"{gas_strategy} must be callable to be a valid gas strategy."
)
else:
gas_strategy = cls.GAS_STRATEGIES[cls.DEFAULT_GAS_STRATEGY]
return gas_strategy
def attach_middleware(self):
chain_id = int(self.client.chain_id)
if self.poa is None: # If POA is not set explicitly, try to autodetect from chain id
self.poa = chain_id in POA_CHAINS
self.log.debug(
f'Ethereum chain: {self.client.chain_name} (chain_id={chain_id}, poa={self.poa})'
)
# For use with Proof-Of-Authority test-blockchains
if self.poa is True:
self.log.debug('Injecting POA middleware at layer 0')
self.client.inject_middleware(geth_poa_middleware, layer=0)
self.client.add_middleware(middleware.time_based_cache_middleware)
# self.client.add_middleware(middleware.latest_block_based_cache_middleware) # TODO: This line causes failed tests and nonce reuse in tests. See #2348.
self.client.add_middleware(middleware.simple_cache_middleware)
self.configure_gas_strategy()
def configure_gas_strategy(self,
gas_strategy: Optional[Callable] = None
) -> None:
if gas_strategy:
reported_gas_strategy = f"fixed/{gas_strategy.name}"
elif isinstance(self.client, InfuraClient):
gas_strategy = construct_datafeed_median_strategy(
speed=self.gas_strategy)
reported_gas_strategy = f"datafeed/{self.gas_strategy}"
else:
reported_gas_strategy = f"web3/{self.gas_strategy}"
gas_strategy = self.get_gas_strategy(self.gas_strategy)
configuration_message = f"Using gas strategy '{reported_gas_strategy}'"
if self.max_gas_price:
__price = Web3.toWei(self.max_gas_price,
'gwei') # from gwei to wei
gas_strategy = max_price_gas_strategy_wrapper(
gas_strategy=gas_strategy, max_gas_price_wei=__price)
configuration_message += f", with a max price of {self.max_gas_price} gwei."
self.client.set_gas_strategy(gas_strategy=gas_strategy)
# TODO: This line must not be called prior to establishing a connection
# Move it down to a lower layer, near the client.
# gwei_gas_price = Web3.fromWei(self.client.gas_price_for_transaction(), 'gwei')
self.log.info(configuration_message)
# self.log.debug(f"Gas strategy currently reports a gas price of {gwei_gas_price} gwei.")
def connect(self):
provider_uri = self.provider_uri
self.log.info(f"Using external Web3 Provider '{self.provider_uri}'")
# Attach Provider
self._attach_provider(provider=self._provider,
provider_uri=provider_uri)
self.log.info("Connecting to {}".format(self.provider_uri))
if self._provider is NO_BLOCKCHAIN_CONNECTION:
raise self.NoProvider(
"There are no configured blockchain providers")
# Connect if not connected
try:
self.w3 = self.Web3(provider=self._provider)
self.client = EthereumClient.from_w3(w3=self.w3)
except requests.ConnectionError: # RPC
raise self.ConnectionFailed(
f'Connection Failed - {str(self.provider_uri)} - is RPC enabled?'
)
except FileNotFoundError: # IPC File Protocol
raise self.ConnectionFailed(
f'Connection Failed - {str(self.provider_uri)} - is IPC enabled?'
)
else:
self.attach_middleware()
return self.is_connected
@property
def provider(self) -> BaseProvider:
return self._provider
def _attach_provider(self,
provider: Optional[BaseProvider] = None,
provider_uri: str = None) -> None:
"""
https://web3py.readthedocs.io/en/latest/providers.html#providers
"""
if not provider_uri and not provider:
raise self.NoProvider("No URI or provider instances supplied.")
if provider_uri and not provider:
uri_breakdown = urlparse(provider_uri)
if uri_breakdown.scheme == 'tester':
providers = {
'pyevm': _get_pyevm_test_provider,
'mock': _get_mock_test_provider
}
provider_scheme = uri_breakdown.netloc
else:
providers = {
'auto': _get_auto_provider,
'ipc': _get_IPC_provider,
'file': _get_IPC_provider,
'ws': _get_websocket_provider,
'wss': _get_websocket_provider,
'http': _get_HTTP_provider,
'https': _get_HTTP_provider,
}
provider_scheme = uri_breakdown.scheme
# auto-detect for file based ipc
if not provider_scheme:
if os.path.exists(provider_uri):
# file is available - assume ipc/file scheme
provider_scheme = 'file'
self.log.info(
f"Auto-detected provider scheme as 'file://' for provider {provider_uri}"
)
try:
self._provider = providers[provider_scheme](provider_uri)
except KeyError:
raise self.UnsupportedProvider(
f"{provider_uri} is an invalid or unsupported blockchain provider URI"
)
else:
self.provider_uri = provider_uri or NO_BLOCKCHAIN_CONNECTION
else:
self._provider = provider
@classmethod
def _handle_failed_transaction(cls,
exception: Exception,
transaction_dict: dict,
contract_function: Union[
ContractFunction, ContractConstructor],
logger: Logger = None) -> None:
"""
Re-raising error handler and context manager for transaction broadcast or
build failure events at the interface layer. This method is a last line of defense
against unhandled exceptions caused by transaction failures and must raise an exception.
# TODO: #1504 - Additional Handling of validation failures (gas limits, invalid fields, etc.)
"""
response = exception.args[0]
# Assume this error is formatted as an RPC response
try:
code = int(response['code'])
message = response['message']
except Exception:
# TODO: #1504 - Try even harder to determine if this is insufficient funds causing the issue,
# This may be best handled at the agent or actor layer for registry and token interactions.
# Worst case scenario - raise the exception held in context implicitly
raise exception
if code != cls.TransactionFailed.IPC_CODE:
# Only handle client-specific exceptions
# https://www.jsonrpc.org/specification Section 5.1
raise exception
if logger:
logger.critical(message) # simple context
transaction_failed = cls.TransactionFailed(
message=message, # rich error (best case)
contract_function=contract_function,
transaction_dict=transaction_dict)
raise transaction_failed from exception
def __log_transaction(self, transaction_dict: dict,
contract_function: ContractFunction):
"""
Format and log a transaction dict and return the transaction name string.
This method *must not* mutate the original transaction dict.
"""
# Do not mutate the original transaction dict
tx = dict(transaction_dict).copy()
# Format
if tx.get('to'):
tx['to'] = to_checksum_address(contract_function.address)
try:
tx['selector'] = contract_function.selector
except AttributeError:
pass
tx['from'] = to_checksum_address(tx['from'])
tx.update({
f: prettify_eth_amount(v)
for f, v in tx.items() if f in ('gasPrice', 'value')
})
payload_pprint = ', '.join("{}: {}".format(k, v)
for k, v in tx.items())
# Log
transaction_name = get_transaction_name(
contract_function=contract_function)
self.log.debug(f"[TX-{transaction_name}] | {payload_pprint}")
@validate_checksum_address
def build_payload(
self,
sender_address: str,
payload: dict = None,
transaction_gas_limit: int = None,
use_pending_nonce: bool = True,
) -> dict:
nonce = self.client.get_transaction_count(account=sender_address,
pending=use_pending_nonce)
base_payload = {
'chainId': int(self.client.chain_id),
'nonce': nonce,
'from': sender_address
}
# Aggregate
if not payload:
payload = {}
payload.update(base_payload)
# Explicit gas override - will skip gas estimation in next operation.
if transaction_gas_limit:
payload['gas'] = int(transaction_gas_limit)
return payload
@validate_checksum_address
def build_contract_transaction(
self,
contract_function: ContractFunction,
sender_address: str,
payload: dict = None,
transaction_gas_limit: Optional[int] = None,
gas_estimation_multiplier: Optional[float] = None,
use_pending_nonce: Optional[bool] = None,
) -> dict:
# Sanity checks for the gas estimation multiplier
if gas_estimation_multiplier is not None:
if not 1 <= gas_estimation_multiplier <= 3: # TODO: Arbitrary upper bound.
raise ValueError(
f"The gas estimation multiplier should be a float between 1 and 3, "
f"but we received {gas_estimation_multiplier}.")
elif transaction_gas_limit is not None:
raise ValueError(
"'transaction_gas_limit' and 'gas_estimation_multiplier' can't be used together."
)
payload = self.build_payload(
sender_address=sender_address,
payload=payload,
transaction_gas_limit=transaction_gas_limit,
use_pending_nonce=use_pending_nonce)
self.__log_transaction(transaction_dict=payload,
contract_function=contract_function)
try:
if 'gas' not in payload:
# As web3 buildTransaction() will estimate gas with block identifier "pending" by default,
# explicitly estimate gas here with block identifier 'latest' if not otherwise specified
# as a pending transaction can cause gas estimation to fail, notably in case of worklock refunds.
payload['gas'] = contract_function.estimateGas(
payload, block_identifier='latest')
transaction_dict = contract_function.buildTransaction(payload)
except (TestTransactionFailed, ValidationError, ValueError) as error:
# Note: Geth (1.9.15) raises ValueError in the same condition that pyevm raises ValidationError here.
# Treat this condition as "Transaction Failed" during gas estimation.
raise self._handle_failed_transaction(
exception=error,
transaction_dict=payload,
contract_function=contract_function,
logger=self.log)
# Overestimate the transaction gas limit according to the gas estimation multiplier, if any
if gas_estimation_multiplier:
gas_estimation = transaction_dict['gas']
overestimation = int(
math.ceil(gas_estimation * gas_estimation_multiplier))
self.log.debug(
f"Gas limit for this TX was increased from {gas_estimation} to {overestimation}, "
f"using a multiplier of {gas_estimation_multiplier}.")
transaction_dict['gas'] = overestimation
# TODO: What if we're going over the block limit? Not likely, but perhaps worth checking (NRN)
return transaction_dict
def sign_and_broadcast_transaction(
self,
transacting_power: TransactingPower,
transaction_dict: TransactionDict,
transaction_name: str = "",
confirmations: int = 0,
fire_and_forget: bool = False) -> Union[TxReceipt, HexBytes]:
"""
Takes a transaction dictionary, signs it with the configured signer, then broadcasts the signed
transaction using the ethereum provider's eth_sendRawTransaction RPC endpoint.
Optionally blocks for receipt and confirmation with 'confirmations', and 'fire_and_forget' flags.
If 'fire and forget' is True this method returns the transaction hash only, without waiting for a receipt -
otherwise return the transaction receipt.
"""
#
# Setup
#
# TODO # 1754 - Move this to singleton - I do not approve... nor does Bogdan?
if GlobalLoggerSettings._json_ipc:
emitter = JSONRPCStdoutEmitter()
else:
emitter = StdoutEmitter()
#
# Sign
#
# TODO: Show the USD Price: https://api.coinmarketcap.com/v1/ticker/ethereum/
price = transaction_dict['gasPrice']
price_gwei = Web3.fromWei(price, 'gwei')
cost_wei = price * transaction_dict['gas']
cost = Web3.fromWei(cost_wei, 'ether')
if transacting_power.is_device:
emitter.message(
f'Confirm transaction {transaction_name} on hardware wallet... '
f'({cost} ETH @ {price_gwei} gwei)',
color='yellow')
signed_raw_transaction = transacting_power.sign_transaction(
transaction_dict)
#
# Broadcast
#
emitter.message(
f'Broadcasting {transaction_name} Transaction ({cost} ETH @ {price_gwei} gwei)',
color='yellow')
try:
txhash = self.client.send_raw_transaction(
signed_raw_transaction) # <--- BROADCAST
emitter.message(f'TXHASH {txhash.hex()}', color='yellow')
except (TestTransactionFailed, ValueError):
raise # TODO: Unify with Transaction failed handling -- Entry point for _handle_failed_transaction
else:
if fire_and_forget:
return txhash
#
# Receipt
#
try: # TODO: Handle block confirmation exceptions
waiting_for = 'receipt'
if confirmations:
waiting_for = f'{confirmations} confirmations'
emitter.message(
f'Waiting {self.TIMEOUT} seconds for {waiting_for}',
color='yellow')
receipt = self.client.wait_for_receipt(txhash,
timeout=self.TIMEOUT,
confirmations=confirmations)
except TimeExhausted:
# TODO: #1504 - Handle transaction timeout
raise
else:
self.log.debug(
f"[RECEIPT-{transaction_name}] | txhash: {receipt['transactionHash'].hex()}"
)
#
# Confirmations
#
# Primary check
transaction_status = receipt.get('status', UNKNOWN_TX_STATUS)
if transaction_status == 0:
failure = f"Transaction transmitted, but receipt returned status code 0. " \
f"Full receipt: \n {pprint.pformat(receipt, indent=2)}"
raise self.InterfaceError(failure)
if transaction_status is UNKNOWN_TX_STATUS:
self.log.info(
f"Unknown transaction status for {txhash} (receipt did not contain a status field)"
)
# Secondary check
tx = self.client.get_transaction(txhash)
if tx["gas"] == receipt["gasUsed"]:
raise self.InterfaceError(
f"Transaction consumed 100% of transaction gas."
f"Full receipt: \n {pprint.pformat(receipt, indent=2)}")
return receipt
@validate_checksum_address
def send_transaction(
self,
contract_function: Union[ContractFunction, ContractConstructor],
transacting_power: TransactingPower,
payload: dict = None,
transaction_gas_limit: Optional[int] = None,
gas_estimation_multiplier: Optional[float] = None,
confirmations: int = 0,
fire_and_forget: bool = False, # do not wait for receipt. See #2385
replace: bool = False,
) -> Union[TxReceipt, HexBytes]:
if fire_and_forget:
if confirmations > 0:
raise ValueError(
"Transaction Prevented: "
"Cannot use 'confirmations' and 'fire_and_forget' options together."
)
use_pending_nonce = False # TODO: #2385
else:
use_pending_nonce = replace # TODO: #2385
transaction = self.build_contract_transaction(
contract_function=contract_function,
sender_address=transacting_power.account,
payload=payload,
transaction_gas_limit=transaction_gas_limit,
gas_estimation_multiplier=gas_estimation_multiplier,
use_pending_nonce=use_pending_nonce)
# Get transaction name
try:
transaction_name = contract_function.fn_name.upper()
except AttributeError:
transaction_name = 'DEPLOY' if isinstance(
contract_function, ContractConstructor) else 'UNKNOWN'
txhash_or_receipt = self.sign_and_broadcast_transaction(
transacting_power=transacting_power,
transaction_dict=transaction,
transaction_name=transaction_name,
confirmations=confirmations,
fire_and_forget=fire_and_forget)
return txhash_or_receipt
def get_contract_by_name(
self,
registry: BaseContractRegistry,
contract_name: str,
contract_version: str = None,
enrollment_version: Union[int, str] = None,
proxy_name: str = None,
use_proxy_address: bool = True) -> VersionedContract:
"""
Instantiate a deployed contract from registry data,
and assimilate it with its proxy if it is upgradeable.
"""
target_contract_records = registry.search(
contract_name=contract_name, contract_version=contract_version)
if not target_contract_records:
raise self.UnknownContract(
f"No such contract records with name {contract_name}:{contract_version}."
)
if proxy_name:
# Lookup proxies; Search for a published proxy that targets this contract record
proxy_records = registry.search(contract_name=proxy_name)
results = list()
for proxy_name, proxy_version, proxy_address, proxy_abi in proxy_records:
proxy_contract = self.client.w3.eth.contract(
abi=proxy_abi,
address=proxy_address,
version=proxy_version,
ContractFactoryClass=self._CONTRACT_FACTORY)
# Read this dispatcher's target address from the blockchain
proxy_live_target_address = proxy_contract.functions.target(
).call()
for target_name, target_version, target_address, target_abi in target_contract_records:
if target_address == proxy_live_target_address:
if use_proxy_address:
triplet = (proxy_address, target_version,
target_abi)
else:
triplet = (target_address, target_version,
target_abi)
else:
continue
results.append(triplet)
if len(results) > 1:
address, _version, _abi = results[0]
message = "Multiple {} deployments are targeting {}".format(
proxy_name, address)
raise self.InterfaceError(message.format(contract_name))
else:
try:
selected_address, selected_version, selected_abi = results[
0]
except IndexError:
raise self.UnknownContract(
f"There are no Dispatcher records targeting '{contract_name}':{contract_version}"
)
else:
# TODO: use_proxy_address doesnt' work in this case. Should we raise if used?
# NOTE: 0 must be allowed as a valid version number
if len(target_contract_records) != 1:
if enrollment_version is None:
m = f"{len(target_contract_records)} records enrolled " \
f"for contract {contract_name}:{contract_version} " \
f"and no version index was supplied."
raise self.InterfaceError(m)
enrollment_version = self.__get_enrollment_version_index(
name=contract_name,
contract_version=contract_version,
version_index=enrollment_version,
enrollments=len(target_contract_records))
else:
enrollment_version = -1 # default
_contract_name, selected_version, selected_address, selected_abi = target_contract_records[
enrollment_version]
# Create the contract from selected sources
unified_contract = self.client.w3.eth.contract(
abi=selected_abi,
address=selected_address,
version=selected_version,
ContractFactoryClass=self._CONTRACT_FACTORY)
return unified_contract
@staticmethod
def __get_enrollment_version_index(version_index: Union[int, str],
enrollments: int, name: str,
contract_version: str):
version_names = {'latest': -1, 'earliest': 0}
try:
version = version_names[version_index]
except KeyError:
try:
version = int(version_index)
except ValueError:
what_is_this = version_index
raise ValueError(
f"'{what_is_this}' is not a valid enrollment version number"
)
else:
if version > enrollments - 1:
message = f"Version index '{version}' is larger than the number of enrollments " \
f"for {name}:{contract_version}."
raise ValueError(message)
return version
def __init__(self,
emitter = None, # TODO # 1754
poa: bool = None,
light: bool = False,
provider_process=NO_PROVIDER_PROCESS,
provider_uri: str = NO_BLOCKCHAIN_CONNECTION,
provider: Web3Providers = NO_BLOCKCHAIN_CONNECTION,
gas_strategy: Union[str, Callable] = DEFAULT_GAS_STRATEGY):
"""
TODO: #1502 - Move to API docs.
Filesystem Configuration Node Client EVM
================ ====================== =============== ===================== ===========================
Solidity Files -- SolidityCompiler - --- HTTPProvider ------ ...
| |
| |
| |
- *BlockchainInterface* -- IPCProvider ----- External EVM (geth, parity...)
| |
| |
TestProvider ----- EthereumTester -------------
|
|
PyEVM (Development Chain)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Runtime Files -- --BlockchainInterface ----> Registry
| | ^
| | |
| | |
Key Files ------ CharacterConfiguration Agent ... (Contract API)
| | ^
| | |
| | |
| | Actor ...Blockchain-Character API)
| | ^
| | |
| | |
Config File --- --------- Character ... (Public API)
^
|
Human
The Blockchain is the junction of the solidity compiler, a contract registry, and a collection of
web3 network providers as a means of interfacing with the ethereum blockchain to execute
or deploy contract code on the network.
Compiler and Registry Usage
-----------------------------
Contracts are freshly re-compiled if an instance of SolidityCompiler is passed; otherwise,
The registry will read contract data saved to disk that is be used to retrieve contact address and op-codes.
Optionally, A registry instance can be passed instead.
Provider Usage
---------------
https: // github.com / ethereum / eth - tester # available-backends
* HTTP Provider - Web3 HTTP provider, typically JSON RPC 2.0 over HTTP
* Websocket Provider - Web3 WS provider, typically JSON RPC 2.0 over WS, supply endpoint uri and websocket=True
* IPC Provider - Web3 File based IPC provider transported over standard I/O
* Custom Provider - A pre-initialized web3.py provider instance to attach to this interface
"""
self.log = Logger('Blockchain')
self.poa = poa
self.provider_uri = provider_uri
self._provider = provider
self._provider_process = provider_process
self.w3 = NO_BLOCKCHAIN_CONNECTION
self.client = NO_BLOCKCHAIN_CONNECTION # type: EthereumClient
self.transacting_power = READ_ONLY_INTERFACE
self.is_light = light
self.gas_strategy = self.get_gas_strategy(gas_strategy)
class EthereumClient:
is_local = False
GETH = 'Geth'
PARITY = 'Parity'
ALT_PARITY = 'Parity-Ethereum'
GANACHE = 'EthereumJS TestRPC'
ETHEREUM_TESTER = 'EthereumTester' # (PyEVM)
CLEF = 'Clef' # Signer-only
BLOCK_CONFIRMATIONS_POLLING_TIME = 3 # seconds
TRANSACTION_POLLING_TIME = 0.5 # seconds
COOLING_TIME = 5 # seconds
STALECHECK_ALLOWABLE_DELAY = 30 # seconds
class ConnectionNotEstablished(RuntimeError):
pass
class SyncTimeout(RuntimeError):
pass
class UnknownAccount(ValueError):
pass
class TransactionBroadcastError(RuntimeError):
pass
class NotEnoughConfirmations(TransactionBroadcastError):
pass
class TransactionTimeout(TransactionBroadcastError):
pass
class ChainReorganizationDetected(TransactionBroadcastError):
"""Raised when block confirmations logic detects that a TX was lost due to a chain reorganization"""
error_message = ("Chain re-organization detected: Transaction {transaction_hash} was reported to be in "
"block {block_hash}, but it's not there anymore")
def __init__(self, receipt):
self.receipt = receipt
self.message = self.error_message.format(transaction_hash=Web3.toHex(receipt['transactionHash']),
block_hash=Web3.toHex(receipt['blockHash']))
super().__init__(self.message)
def __init__(self,
w3,
node_technology: str,
version: str,
platform: str,
backend: str):
self.w3 = w3
self.node_technology = node_technology
self.node_version = version
self.platform = platform
self.backend = backend
self.log = Logger(self.__class__.__name__)
self._add_default_middleware()
def _add_default_middleware(self):
# default retry functionality
self.log.debug('Adding RPC retry middleware to client')
self.add_middleware(RetryRequestMiddleware)
@classmethod
def _get_variant(cls, w3):
return cls
@classmethod
def from_w3(cls, w3: Web3) -> 'EthereumClient':
"""
Client version strings:
Geth -> 'Geth/v1.4.11-stable-fed692f6/darwin/go1.7'
Parity -> 'Parity-Ethereum/v2.5.1-beta-e0141f8-20190510/x86_64-linux-gnu/rustc1.34.1'
Ganache -> 'EthereumJS TestRPC/v2.1.5/ethereum-js'
PyEVM -> 'EthereumTester/0.1.0b39/linux/python3.6.7'
"""
clients = {
# Geth
cls.GETH: GethClient,
# Parity
cls.PARITY: ParityClient,
cls.ALT_PARITY: ParityClient,
# Test Clients
cls.GANACHE: GanacheClient,
cls.ETHEREUM_TESTER: EthereumTesterClient,
}
try:
client_data = w3.clientVersion.split('/')
node_technology = client_data[0]
ClientSubclass = clients[node_technology]
except (ValueError, IndexError):
raise ValueError(f"Invalid client version string. Got '{w3.clientVersion}'")
except KeyError:
raise NotImplementedError(f'{w3.clientVersion} is not a supported ethereum client')
client_kwargs = {
'node_technology': node_technology,
'version': client_data[1],
'backend': client_data[-1],
'platform': client_data[2] if len(client_data) == 4 else None # Platform is optional
}
instance = ClientSubclass._get_variant(w3)(w3, **client_kwargs)
return instance
@property
def peers(self):
raise NotImplementedError
@property
def chain_name(self) -> str:
chain_inventory = LOCAL_CHAINS if self.is_local else PUBLIC_CHAINS
name = chain_inventory.get(self.chain_id, UNKNOWN_DEVELOPMENT_CHAIN_ID)
return name
def lock_account(self, account) -> bool:
if self.is_local:
return True
return NotImplemented
def unlock_account(self, account, password, duration=None) -> bool:
if self.is_local:
return True
return NotImplemented
@property
def is_connected(self):
return self.w3.isConnected()
@property
def etherbase(self) -> str:
return self.w3.eth.accounts[0]
@property
def accounts(self):
return self.w3.eth.accounts
def get_balance(self, account):
return self.w3.eth.getBalance(account)
def inject_middleware(self, middleware, **kwargs):
self.w3.middleware_onion.inject(middleware, **kwargs)
def add_middleware(self, middleware):
self.w3.middleware_onion.add(middleware)
def set_gas_strategy(self, gas_strategy):
self.w3.eth.setGasPriceStrategy(gas_strategy)
@property
def chain_id(self) -> int:
try:
# from hex-str
return int(self.w3.eth.chainId, 16)
except TypeError:
# from str
return int(self.w3.eth.chainId)
@property
def net_version(self) -> int:
return int(self.w3.net.version)
def get_contract(self, **kwargs) -> Contract:
return self.w3.eth.contract(**kwargs)
@property
def gas_price(self) -> Wei:
"""
Returns client's gas price. Underneath, it uses the eth_gasPrice JSON-RPC method
"""
return self.w3.eth.gasPrice
def gas_price_for_transaction(self, transaction=None) -> Wei:
"""
Obtains a gas price via the current gas strategy, if any; otherwise, it resorts to the client's gas price.
This method mirrors the behavior of web3._utils.transactions when building transactions.
"""
return self.w3.eth.generateGasPrice(transaction) or self.gas_price
@property
def block_number(self) -> BlockNumber:
return self.w3.eth.blockNumber
@property
def coinbase(self) -> ChecksumAddress:
return self.w3.eth.coinbase
def wait_for_receipt(self,
transaction_hash: str,
timeout: float,
confirmations: int = 0) -> TxReceipt:
receipt: TxReceipt = None
if confirmations:
# If we're waiting for confirmations, we may as well let pass some time initially to make everything easier
time.sleep(self.COOLING_TIME)
# We'll keep trying to get receipts until there are enough confirmations or the timeout happens
with Timeout(seconds=timeout, exception=self.TransactionTimeout) as timeout_context:
while not receipt:
try:
receipt = self.block_until_enough_confirmations(transaction_hash=transaction_hash,
timeout=timeout,
confirmations=confirmations)
except (self.ChainReorganizationDetected, self.NotEnoughConfirmations, TimeExhausted):
timeout_context.sleep(self.BLOCK_CONFIRMATIONS_POLLING_TIME)
continue
else:
# If not asking for confirmations, just use web3 and assume the returned receipt is final
try:
receipt = self.w3.eth.waitForTransactionReceipt(transaction_hash=transaction_hash,
timeout=timeout,
poll_latency=self.TRANSACTION_POLLING_TIME)
except TimeExhausted:
raise # TODO: #1504 - Handle transaction timeout
return receipt
def block_until_enough_confirmations(self, transaction_hash: str, timeout: float, confirmations: int) -> dict:
receipt: TxReceipt = self.w3.eth.waitForTransactionReceipt(transaction_hash=transaction_hash,
timeout=timeout,
poll_latency=self.TRANSACTION_POLLING_TIME)
preliminary_block_hash = Web3.toHex(receipt['blockHash'])
tx_block_number = Web3.toInt(receipt['blockNumber'])
self.log.info(f"Transaction {Web3.toHex(transaction_hash)} is preliminarily included in "
f"block {preliminary_block_hash}")
confirmations_timeout = self._calculate_confirmations_timeout(confirmations)
confirmations_so_far = 0
with Timeout(seconds=confirmations_timeout, exception=self.NotEnoughConfirmations) as timeout_context:
while confirmations_so_far < confirmations:
timeout_context.sleep(self.BLOCK_CONFIRMATIONS_POLLING_TIME)
self.check_transaction_is_on_chain(receipt=receipt)
confirmations_so_far = self.block_number - tx_block_number
self.log.info(f"We have {confirmations_so_far} confirmations. "
f"Waiting for {confirmations - confirmations_so_far} more.")
return receipt
@staticmethod
def _calculate_confirmations_timeout(confirmations):
confirmations_timeout = 3 * AVERAGE_BLOCK_TIME_IN_SECONDS * confirmations
return confirmations_timeout
def check_transaction_is_on_chain(self, receipt: TxReceipt) -> bool:
transaction_hash = Web3.toHex(receipt['transactionHash'])
try:
new_receipt = self.w3.eth.getTransactionReceipt(transaction_hash)
except TransactionNotFound:
reorg_detected = True
else:
reorg_detected = receipt['blockHash'] != new_receipt['blockHash']
if reorg_detected:
exception = self.ChainReorganizationDetected(receipt=receipt)
self.log.info(exception.message)
raise exception
# TODO: Consider adding an optional param in this exception to include extra info (e.g. new block)
return True
def sign_transaction(self, transaction_dict: dict) -> bytes:
raise NotImplementedError
def get_transaction(self, transaction_hash) -> dict:
return self.w3.eth.getTransaction(transaction_hash)
def get_transaction_receipt(self, transaction_hash) -> Union[dict, None]:
return self.w3.eth.getTransactionReceipt(transaction_hash)
def get_transaction_count(self, account: str, pending: bool) -> int:
block_identifier = 'pending' if pending else 'latest'
return self.w3.eth.getTransactionCount(account, block_identifier)
def send_transaction(self, transaction_dict: dict) -> str:
return self.w3.eth.sendTransaction(transaction_dict)
def send_raw_transaction(self, transaction_bytes: bytes) -> str:
return self.w3.eth.sendRawTransaction(transaction_bytes)
def sign_message(self, account: str, message: bytes) -> str:
"""
Calls the appropriate signing function for the specified account on the
backend. If the backend is based on eth-tester, then it uses the
eth-tester signing interface to do so.
"""
return self.w3.eth.sign(account, data=message)
def get_blocktime(self):
highest_block = self.w3.eth.getBlock('latest')
now = highest_block['timestamp']
return now
def _has_latest_block(self) -> bool:
# TODO: Investigate using `web3.middleware.make_stalecheck_middleware` #2060
# check that our local chain data is up to date
return (time.time() - self.get_blocktime()) < self.STALECHECK_ALLOWABLE_DELAY
def parse_transaction_data(self, transaction):
return transaction.input
class CharacterConfiguration(BaseConfiguration):
"""
'Sideways Engagement' of Character classes; a reflection of input parameters.
"""
VERSION = 2 # bump when static payload scheme changes
CHARACTER_CLASS = NotImplemented
DEFAULT_CONTROLLER_PORT = NotImplemented
DEFAULT_DOMAIN = NetworksInventory.DEFAULT
DEFAULT_NETWORK_MIDDLEWARE = RestMiddleware
TEMP_CONFIGURATION_DIR_PREFIX = 'tmp-nucypher'
# When we begin to support other threshold schemes, this will be one of the concepts that makes us want a factory. #571
known_node_class = Ursula
# Gas
DEFAULT_GAS_STRATEGY = 'fast'
_CONFIG_FIELDS = ('config_root', 'poa', 'light', 'provider_uri',
'registry_filepath', 'gas_strategy', 'signer_uri')
def __init__(
self,
# Base
emitter=None,
config_root: str = None,
filepath: str = None,
# Mode
dev_mode: bool = False,
federated_only: bool = False,
# Identity
checksum_address: str = None,
crypto_power: CryptoPower = None,
# Keyring
keyring: NucypherKeyring = None,
keyring_root: str = None,
# Learner
learn_on_same_thread: bool = False,
abort_on_learning_error: bool = False,
start_learning_now: bool = True,
# Network
controller_port: int = None,
domain: str = DEFAULT_DOMAIN,
interface_signature: Signature = None,
network_middleware: RestMiddleware = None,
lonely: bool = False,
# Node Storage
known_nodes: set = None,
node_storage: NodeStorage = None,
reload_metadata: bool = True,
save_metadata: bool = True,
# Blockchain
poa: bool = None,
light: bool = False,
sync: bool = False,
provider_uri: str = None,
provider_process=None,
gas_strategy: Union[Callable, str] = DEFAULT_GAS_STRATEGY,
signer_uri: str = None,
# Registry
registry: BaseContractRegistry = None,
registry_filepath: str = None,
# Deployed Workers
worker_data: dict = None):
self.log = Logger(self.__class__.__name__)
UNINITIALIZED_CONFIGURATION.bool_value(False)
# Identity
# NOTE: NodeConfigurations can only be used with Self-Characters
self.is_me = True
self.checksum_address = checksum_address
# Keyring
self.crypto_power = crypto_power
self.keyring = keyring or NO_KEYRING_ATTACHED
self.keyring_root = keyring_root or UNINITIALIZED_CONFIGURATION
# Contract Registry
if registry and registry_filepath:
if registry.filepath != registry_filepath:
error = f"Inconsistent registry filepaths for '{registry.filepath}' and '{registry_filepath}'."
raise ValueError(error)
else:
self.log.warn(
f"Registry and registry filepath were both passed.")
self.registry = registry or NO_BLOCKCHAIN_CONNECTION.bool_value(False)
self.registry_filepath = registry_filepath or UNINITIALIZED_CONFIGURATION
# Blockchain
self.poa = poa
self.is_light = light
self.provider_uri = provider_uri or NO_BLOCKCHAIN_CONNECTION
self.provider_process = provider_process or NO_BLOCKCHAIN_CONNECTION
self.signer_uri = signer_uri or None
# Learner
self.federated_only = federated_only
self.domain = domain
self.learn_on_same_thread = learn_on_same_thread
self.abort_on_learning_error = abort_on_learning_error
self.start_learning_now = start_learning_now
self.save_metadata = save_metadata
self.reload_metadata = reload_metadata
self.known_nodes = known_nodes or set() # handpicked
self.lonely = lonely
# Configuration
self.__dev_mode = dev_mode
self.config_file_location = filepath or UNINITIALIZED_CONFIGURATION
self.config_root = UNINITIALIZED_CONFIGURATION
# Deployed Workers
self.worker_data = worker_data
#
# Federated vs. Blockchain arguments consistency
#
#
# Federated
#
if self.federated_only:
# Check for incompatible values
blockchain_args = {
'filepath': registry_filepath,
'poa': poa,
'provider_process': provider_process,
'provider_uri': provider_uri,
'gas_strategy': gas_strategy
}
if any(blockchain_args.values()):
bad_args = ", ".join(f"{arg}={val}"
for arg, val in blockchain_args.items()
if val)
self.log.warn(
f"Arguments {bad_args} are incompatible with federated_only. "
f"Overridden with a sane default.")
# Clear decentralized attributes to ensure consistency with a
# federated configuration.
self.poa = False
self.is_light = False
self.provider_uri = None
self.provider_process = None
self.registry_filepath = None
self.gas_strategy = None
#
# Decentralized
#
else:
self.gas_strategy = gas_strategy
is_initialized = BlockchainInterfaceFactory.is_interface_initialized(
provider_uri=self.provider_uri)
if not is_initialized and provider_uri:
BlockchainInterfaceFactory.initialize_interface(
provider_uri=self.provider_uri,
poa=self.poa,
light=self.is_light,
provider_process=self.provider_process,
sync=sync,
emitter=emitter,
gas_strategy=gas_strategy)
else:
self.log.warn(
f"Using existing blockchain interface connection ({self.provider_uri})."
)
if not self.registry:
# TODO: These two code blocks are untested.
if not self.registry_filepath: # TODO: Registry URI (goerli://speedynet.json) :-)
self.log.info(f"Fetching latest registry from source.")
self.registry = InMemoryContractRegistry.from_latest_publication(
network=self.domain)
else:
self.registry = LocalContractRegistry(
filepath=self.registry_filepath)
self.log.info(f"Using local registry ({self.registry}).")
if dev_mode:
self.__temp_dir = UNINITIALIZED_CONFIGURATION
self.__setup_node_storage()
self.initialize(password=DEVELOPMENT_CONFIGURATION)
else:
self.__temp_dir = LIVE_CONFIGURATION
self.config_root = config_root or self.DEFAULT_CONFIG_ROOT
self._cache_runtime_filepaths()
self.__setup_node_storage(node_storage=node_storage)
# Network
self.controller_port = controller_port or self.DEFAULT_CONTROLLER_PORT
self.network_middleware = network_middleware or self.DEFAULT_NETWORK_MIDDLEWARE(
registry=self.registry)
self.interface_signature = interface_signature
super().__init__(filepath=self.config_file_location,
config_root=self.config_root)
def __call__(self, **character_kwargs):
return self.produce(**character_kwargs)
@classmethod
def checksum_address_from_filepath(cls, filepath: str) -> str:
pattern = re.compile(
r'''
(^\w+)-
(0x{1} # Then, 0x the start of the string, exactly once
[0-9a-fA-F]{40}) # Followed by exactly 40 hex chars
''', re.VERBOSE)
filename = os.path.basename(filepath)
match = pattern.match(filename)
if match:
character_name, checksum_address = match.groups()
else:
# Extract from default by "peeking" inside the configuration file.
default_name = cls.generate_filename()
if filename == default_name:
checksum_address = cls.peek(filepath=filepath,
field='checksum_address')
###########
# TODO: Cleanup and deprecate worker_address in config files, leaving only checksum_address
from nucypher.config.characters import UrsulaConfiguration
if isinstance(cls, UrsulaConfiguration):
federated = bool(
cls.peek(filepath=filepath, field='federated_only'))
if not federated:
checksum_address = cls.peek(filepath=cls.filepath,
field='worker_address')
###########
else:
raise ValueError(
f"Cannot extract checksum from filepath '{filepath}'")
if not is_checksum_address(checksum_address):
raise RuntimeError(
f"Invalid checksum address detected in configuration file at '{filepath}'."
)
return checksum_address
def update(self, **kwargs) -> None:
"""
A facility for updating existing attributes on existing configuration instances.
Warning: This method allows mutation and may result in an inconsistent configuration.
"""
return super().update(modifier=self.checksum_address,
filepath=self.config_file_location,
**kwargs)
@classmethod
def generate(cls, password: str, *args, **kwargs):
"""Shortcut: Hook-up a new initial installation and write configuration file to the disk"""
node_config = cls(dev_mode=False, *args, **kwargs)
node_config.initialize(password=password)
node_config.to_configuration_file()
return node_config
def cleanup(self) -> None:
if self.__dev_mode:
self.__temp_dir.cleanup()
@property
def dev_mode(self) -> bool:
return self.__dev_mode
def __setup_node_storage(self, node_storage=None) -> None:
if self.dev_mode:
node_storage = ForgetfulNodeStorage(
registry=self.registry, federated_only=self.federated_only)
elif not node_storage:
node_storage = LocalFileBasedNodeStorage(
registry=self.registry,
config_root=self.config_root,
federated_only=self.federated_only)
self.node_storage = node_storage
def forget_nodes(self) -> None:
self.node_storage.clear()
message = "Removed all stored node node metadata and certificates"
self.log.debug(message)
def destroy(self) -> None:
"""Parse a node configuration and remove all associated files from the filesystem"""
self.attach_keyring()
self.keyring.destroy()
os.remove(self.config_file_location)
def generate_parameters(self, **overrides) -> dict:
"""
Warning: This method allows mutation and may result in an inconsistent configuration.
"""
merged_parameters = {
**self.static_payload(),
**self.dynamic_payload,
**overrides
}
character_init_params = filter(
lambda t: t[0] not in self._CONFIG_FIELDS,
merged_parameters.items())
return dict(character_init_params)
def produce(self, **overrides) -> CHARACTER_CLASS:
"""Initialize a new character instance and return it."""
merged_parameters = self.generate_parameters(**overrides)
character = self.CHARACTER_CLASS(**merged_parameters)
return character
@classmethod
def assemble(cls, filepath: str = None, **overrides) -> dict:
"""
Warning: This method allows mutation and may result in an inconsistent configuration.
"""
payload = cls._read_configuration_file(filepath=filepath)
node_storage = cls.load_node_storage(
storage_payload=payload['node_storage'],
federated_only=payload['federated_only'])
domain = payload['domain']
# Assemble
payload.update(dict(node_storage=node_storage, domain=domain))
# Filter out None values from **overrides to detect, well, overrides...
# Acts as a shim for optional CLI flags.
overrides = {k: v for k, v in overrides.items() if v is not None}
payload = {**payload, **overrides}
return payload
@classmethod
def from_configuration_file(
cls,
filepath: str = None,
provider_process=None,
**overrides # < ---- Inlet for CLI Flags
) -> 'CharacterConfiguration':
"""Initialize a CharacterConfiguration from a JSON file."""
filepath = filepath or cls.default_filepath()
assembled_params = cls.assemble(filepath=filepath, **overrides)
node_configuration = cls(filepath=filepath,
provider_process=provider_process,
**assembled_params)
return node_configuration
def validate(self) -> bool:
# Top-level
if not os.path.exists(self.config_root):
raise self.ConfigurationError(
f'No configuration directory found at {self.config_root}.')
# Sub-paths
filepaths = self.runtime_filepaths
for field, path in filepaths.items():
if path and not os.path.exists(path):
message = 'Missing configuration file or directory: {}.'
if 'registry' in path:
message += ' Did you mean to pass --federated-only?'
raise CharacterConfiguration.InvalidConfiguration(
message.format(path))
return True
def static_payload(self) -> dict:
"""Exported static configuration values for initializing Ursula"""
payload = dict(
# Identity
federated_only=self.federated_only,
checksum_address=self.checksum_address,
keyring_root=self.keyring_root,
# Behavior
domain=self.domain,
learn_on_same_thread=self.learn_on_same_thread,
abort_on_learning_error=self.abort_on_learning_error,
start_learning_now=self.start_learning_now,
save_metadata=self.save_metadata,
node_storage=self.node_storage.payload(),
lonely=self.lonely,
)
# Optional values (mode)
if not self.federated_only:
if self.provider_uri:
if not self.signer_uri:
self.signer_uri = self.provider_uri
payload.update(
dict(provider_uri=self.provider_uri,
poa=self.poa,
light=self.is_light,
signer_uri=self.signer_uri))
if self.registry_filepath:
payload.update(dict(registry_filepath=self.registry_filepath))
# Gas Price
payload.update(dict(gas_strategy=self.gas_strategy))
# Merge with base payload
base_payload = super().static_payload()
base_payload.update(payload)
return payload
@property # TODO: Graduate to a method and "derive" dynamic from static payload.
def dynamic_payload(self) -> dict:
"""Exported dynamic configuration values for initializing Ursula"""
payload = dict()
if not self.federated_only:
testnet = self.domain != NetworksInventory.MAINNET
signer = Signer.from_signer_uri(self.signer_uri, testnet=testnet)
payload.update(dict(registry=self.registry, signer=signer))
payload.update(
dict(network_middleware=self.network_middleware
or self.DEFAULT_NETWORK_MIDDLEWARE(),
known_nodes=self.known_nodes,
node_storage=self.node_storage,
crypto_power_ups=self.derive_node_power_ups()))
return payload
def generate_filepath(self,
filepath: str = None,
modifier: str = None,
override: bool = False) -> str:
modifier = modifier or self.checksum_address
filepath = super().generate_filepath(filepath=filepath,
modifier=modifier,
override=override)
return filepath
@property
def runtime_filepaths(self) -> dict:
filepaths = dict(config_root=self.config_root,
keyring_root=self.keyring_root,
registry_filepath=self.registry_filepath)
return filepaths
@classmethod
def generate_runtime_filepaths(cls, config_root: str) -> dict:
"""Dynamically generate paths based on configuration root directory"""
filepaths = dict(config_root=config_root,
config_file_location=os.path.join(
config_root, cls.generate_filename()),
keyring_root=os.path.join(config_root, 'keyring'))
return filepaths
def _cache_runtime_filepaths(self) -> None:
"""Generate runtime filepaths and cache them on the config object"""
filepaths = self.generate_runtime_filepaths(
config_root=self.config_root)
for field, filepath in filepaths.items():
if getattr(self, field) is UNINITIALIZED_CONFIGURATION:
setattr(self, field, filepath)
def attach_keyring(self,
checksum_address: str = None,
*args,
**kwargs) -> None:
account = checksum_address or self.checksum_address
if not account:
raise self.ConfigurationError(
"No account specified to unlock keyring")
if self.keyring is not NO_KEYRING_ATTACHED:
if self.keyring.checksum_address != account:
raise self.ConfigurationError(
"There is already a keyring attached to this configuration."
)
return
self.keyring = NucypherKeyring(keyring_root=self.keyring_root,
account=account,
*args,
**kwargs)
def derive_node_power_ups(self) -> List[CryptoPowerUp]:
power_ups = list()
if self.is_me and not self.dev_mode:
for power_class in self.CHARACTER_CLASS._default_crypto_powerups:
power_up = self.keyring.derive_crypto_power(power_class)
power_ups.append(power_up)
return power_ups
def initialize(self, password: str) -> str:
"""Initialize a new configuration and write installation files to disk."""
# Development
if self.dev_mode:
self.__temp_dir = TemporaryDirectory(
prefix=self.TEMP_CONFIGURATION_DIR_PREFIX)
self.config_root = self.__temp_dir.name
# Persistent
else:
self._ensure_config_root_exists()
self.write_keyring(password=password)
self._cache_runtime_filepaths()
self.node_storage.initialize()
# Validate
if not self.__dev_mode:
self.validate()
# Success
message = "Created nucypher installation files at {}".format(
self.config_root)
self.log.debug(message)
return self.config_root
def write_keyring(self,
password: str,
checksum_address: str = None,
**generation_kwargs) -> NucypherKeyring:
if self.federated_only:
checksum_address = FEDERATED_ADDRESS
elif not checksum_address:
# Note: It is assumed the blockchain interface is not yet connected.
if self.provider_process:
# Generate Geth's "datadir"
if not os.path.exists(self.provider_process.data_dir):
os.mkdir(self.provider_process.data_dir)
# Get or create wallet address
if not self.checksum_address:
self.checksum_address = self.provider_process.ensure_account_exists(
password=password)
elif self.checksum_address not in self.provider_process.accounts(
):
raise self.ConfigurationError(
f'Unknown Account {self.checksum_address}')
elif not self.checksum_address:
raise self.ConfigurationError(
f'No checksum address provided for decentralized configuration.'
)
checksum_address = self.checksum_address
self.keyring = NucypherKeyring.generate(
password=password,
keyring_root=self.keyring_root,
checksum_address=checksum_address,
**generation_kwargs)
if self.federated_only:
self.checksum_address = self.keyring.checksum_address
return self.keyring
@classmethod
def load_node_storage(cls, storage_payload: dict, federated_only: bool):
from nucypher.config.storages import NodeStorage
node_storage_subclasses = {
storage._name: storage
for storage in NodeStorage.__subclasses__()
}
storage_type = storage_payload[NodeStorage._TYPE_LABEL]
storage_class = node_storage_subclasses[storage_type]
node_storage = storage_class.from_payload(
payload=storage_payload, federated_only=federated_only)
return node_storage
from typing import Sequence
import requests
from constant_sorrow.constants import EXEMPT_FROM_VERIFICATION
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.x509 import Certificate
from nucypher_core import MetadataRequest, FleetStateChecksum, NodeMetadata
from requests.exceptions import SSLError
from nucypher.blockchain.eth.registry import BaseContractRegistry
from nucypher.config.storages import ForgetfulNodeStorage
from nucypher.network.exceptions import NodeSeemsToBeDown
from nucypher.utilities.logging import Logger
SSL_LOGGER = Logger('ssl-middleware')
EXEMPT_FROM_VERIFICATION.bool_value(False)
class NucypherMiddlewareClient:
library = requests
timeout = 1.2
def __init__(self,
registry: Optional['BaseContractRegistry'] = None,
eth_provider_uri: Optional[str] = None,
storage: Optional['NodeStorage'] = None,
*args,
**kwargs):
self.registry = registry
class FleetSensor:
"""
A representation of a fleet of NuCypher nodes.
If `this_node` is provided, it will be included in the state checksum
(but not returned during iteration/lookups).
"""
log = Logger("Learning")
def __init__(self, domain: str, this_node: Optional['Ursula'] = None):
self._domain = domain
self._current_state = FleetState.new(this_node)
self._archived_states = [self._current_state.archived()]
self.remote_states = {}
# temporary accumulator for new nodes to avoid updating the fleet state every time
self._nodes_to_add = set()
self._nodes_to_remove = set() # Beginning of bucketing.
self._auto_update_state = False
def record_node(self, node: 'Ursula'):
if node.domain == self._domain:
self._nodes_to_add.add(node)
if self._auto_update_state:
self.log.info(f"Updating fleet state after saving node {node}")
self.record_fleet_state()
else:
msg = f"Rejected node {node} because its domain is '{node.domain}' but we're only tracking '{self._domain}'"
self.log.warn(msg)
def __getitem__(self, item):
return self._current_state[item]
def __bool__(self):
return bool(self._current_state)
def __contains__(self, item):
"""
Checks if the node *with the same metadata* is recorded in the current state.
Does not compare ``item`` with the owner node of this FleetSensor.
"""
return item in self._current_state
def __iter__(self):
yield from self._current_state
def __len__(self):
return len(self._current_state)
def __repr__(self):
return f"FleetSensor({self._current_state.__repr__()})"
@property
def current_state(self):
return self._current_state
@property
def checksum(self):
return self._current_state.checksum
@property
def population(self):
return self._current_state.population
@property
def nickname(self):
return self._current_state.nickname
@property
def icon(self) -> str:
return self._current_state.icon
@property
def timestamp(self):
return self._current_state.timestamp
def items(self):
return self._current_state.items()
def values(self):
return self._current_state.values()
def latest_states(self, quantity: int) -> List[ArchivedFleetState]:
"""
Returns at most ``quantity`` latest archived states (including the current one),
in chronological order.
"""
latest = self._archived_states[-min(len(self._archived_states), quantity):]
return latest
def addresses(self):
return self._current_state.addresses()
def snapshot(self):
return self._current_state.snapshot()
@staticmethod
def unpack_snapshot(data):
return FleetState.unpack_snapshot(data)
def record_fleet_state(self, skip_this_node: bool = False):
new_state = self._current_state.with_updated_nodes(nodes_to_add=self._nodes_to_add,
nodes_to_remove=self._nodes_to_remove,
skip_this_node=skip_this_node)
self._nodes_to_add = set()
self._nodes_to_remove = set()
self._current_state = new_state
# TODO: set a limit on the number of archived states?
# Two ways to collect archived states:
# 1. (current) add a state to the archive every time it changes
# 2. (possible) keep a dictionary of known states
# and bump the timestamp of a previously encountered one
if new_state.checksum != self._archived_states[-1].checksum:
self._archived_states.append(new_state.archived())
def shuffled(self):
return self._current_state.shuffled()
def mark_as(self, label: Exception, node: 'Ursula'):
# TODO: for now we're not using `label` in any way, so we're just ignoring it
self._nodes_to_remove.add(node.checksum_address)
def record_remote_fleet_state(self,
checksum_address: ChecksumAddress,
state_checksum: str,
timestamp: maya.MayaDT,
population: int):
nickname = Nickname.from_seed(state_checksum, length=1)
self.remote_states[checksum_address] = ArchivedFleetState(checksum=state_checksum,
nickname=nickname,
timestamp=timestamp,
population=population)
class NodeEngagementMutex:
"""
TODO: Does this belong on middleware?
TODO: There are a couple of ways this can break. If one fo the jobs hangs, the whole thing will hang. Also,
if there are fewer successfully completed than percent_to_complete_before_release, the partial queue will never
release.
TODO: Make registry per... I guess Policy? It's weird to be able to accidentally enact again.
"""
log = Logger("Policy")
def __init__(
self,
callable_to_engage, # TODO: typing.Protocol
nodes,
network_middleware,
percent_to_complete_before_release=5,
note=None,
threadpool_size=120,
timeout=20,
*args,
**kwargs):
self.f = callable_to_engage
self.nodes = nodes
self.network_middleware = network_middleware
self.args = args
self.kwargs = kwargs
self.completed = {}
self.failed = {}
self._started = False
self._finished = False
self.timeout = timeout
self.percent_to_complete_before_release = percent_to_complete_before_release
self._partial_queue = Queue()
self._completion_queue = Queue()
self._block_until_this_many_are_complete = math.ceil(
len(nodes) * self.percent_to_complete_before_release / 100)
self.nodes_contacted_during_partial_block = False
self.when_complete = Deferred(
) # TODO: Allow cancelling via KB Interrupt or some other way?
if note is None:
self._repr = f"{callable_to_engage} to {len(nodes)} nodes"
else:
self._repr = f"{note}: {callable_to_engage} to {len(nodes)} nodes"
self._threadpool = ThreadPool(minthreads=threadpool_size,
maxthreads=threadpool_size,
name=self._repr)
self.log.info(f"NEM spinning up {self._threadpool}")
self._threadpool.callInThread(self._bail_on_timeout)
def __repr__(self):
return self._repr
def _bail_on_timeout(self):
while True:
if self.when_complete.called:
return
duration = datetime.datetime.now() - self._started
if duration.seconds >= self.timeout:
try:
self._threadpool.stop()
except AlreadyQuit:
raise RuntimeError(
"Is there a race condition here? If this line is being hit, it's a bug."
)
raise RuntimeError(
f"Timed out. Nodes completed: {self.completed}")
time.sleep(.5)
def block_until_success_is_reasonably_likely(self):
"""
https://www.youtube.com/watch?v=OkSLswPSq2o
"""
if len(self.completed) < self._block_until_this_many_are_complete:
try:
completed_for_reasonable_likelihood_of_success = self._partial_queue.get(
timeout=self.timeout) # TODO: Shorter timeout here?
except Empty:
raise RuntimeError(
f"Timed out. Nodes completed: {self.completed}")
self.log.debug(
f"{len(self.completed)} nodes were contacted while blocking for a little while."
)
return completed_for_reasonable_likelihood_of_success
else:
return self.completed
def block_until_complete(self):
if self.total_disposed() < len(self.nodes):
try:
_ = self._completion_queue.get(
timeout=self.timeout
) # Interesting opportuntiy to pass some data, like the list of contacted nodes above.
except Empty:
raise RuntimeError(
f"Timed out. Nodes completed: {self.completed}")
if not reactor.running and not self._threadpool.joined:
# If the reactor isn't running, the user *must* call this, because this is where we stop.
self._threadpool.stop()
def _handle_success(self, response, node):
if response.status_code == 201:
self.completed[node] = response
else:
assert False # TODO: What happens if this is a 300 or 400 level response? (A 500 response will propagate as an error and be handled in the errback chain.)
if self.nodes_contacted_during_partial_block:
self._consider_finalizing()
else:
if len(self.completed) >= self._block_until_this_many_are_complete:
contacted = tuple(self.completed.keys())
self.nodes_contacted_during_partial_block = contacted
self.log.debug(
f"Blocked for a little while, completed {contacted} nodes")
self._partial_queue.put(contacted)
return response
def _handle_error(self, failure, node):
self.failed[node] = failure # TODO: Add a failfast mode?
self._consider_finalizing()
self.log.warn(f"{node} failed: {failure}")
def total_disposed(self):
return len(self.completed) + len(self.failed)
def _consider_finalizing(self):
if not self._finished:
if self.total_disposed() == len(self.nodes):
# TODO: Consider whether this can possibly hang.
self._finished = True
if reactor.running:
reactor.callInThread(self._threadpool.stop)
self._completion_queue.put(self.completed)
self.when_complete.callback(self.completed)
self.log.info(f"{self} finished.")
else:
raise RuntimeError("Already finished.")
def _engage_node(self, node):
maybe_coro = self.f(node,
network_middleware=self.network_middleware,
*self.args,
**self.kwargs)
d = ensureDeferred(maybe_coro)
d.addCallback(self._handle_success, node)
d.addErrback(self._handle_error, node)
return d
def start(self):
if self._started:
raise RuntimeError("Already started.")
self._started = datetime.datetime.now()
self.log.info(f"NEM Starting {self._threadpool}")
for node in self.nodes:
self._threadpool.callInThread(self._engage_node, node)
self._threadpool.start()
def make_rest_app(
db_filepath: str,
this_node,
serving_domains,
log=Logger("http-application-layer")) -> Tuple:
forgetful_node_storage = ForgetfulNodeStorage(
federated_only=this_node.federated_only)
from nucypher.datastore import datastore
log.info("Starting datastore {}".format(db_filepath))
datastore = datastore.Datastore(db_filepath)
from nucypher.characters.lawful import Alice, Ursula
_alice_class = Alice
_node_class = Ursula
rest_app = Flask("ursula-service")
rest_app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD_CONTENT_LENGTH
@rest_app.route("/public_information")
def public_information():
"""
REST endpoint for public keys and address.
"""
response = Response(response=bytes(this_node),
mimetype='application/octet-stream')
return response
@rest_app.route("/ping", methods=['POST'])
def ping():
"""
Asks this node: "Can you access my public information endpoint"?
"""
try:
requesting_ursula = Ursula.from_bytes(request.data,
registry=this_node.registry)
requesting_ursula.mature()
except ValueError: # (ValueError)
return Response({'error': 'Invalid Ursula'}, status=400)
else:
initiator_address, initiator_port = tuple(
requesting_ursula.rest_interface)
# Compare requester and posted Ursula information
request_address = request.environ['REMOTE_ADDR']
if request_address != initiator_address:
return Response({'error': 'Suspicious origin address'}, status=400)
#
# Make a Sandwich
#
try:
# Fetch and store initiator's teacher certificate.
certificate = this_node.network_middleware.get_certificate(
host=initiator_address, port=initiator_port)
certificate_filepath = this_node.node_storage.store_node_certificate(
certificate=certificate)
requesting_ursula_bytes = this_node.network_middleware.client.node_information(
host=initiator_address,
port=initiator_port,
certificate_filepath=certificate_filepath)
except NodeSeemsToBeDown:
return Response({'error': 'Unreachable node'},
status=400) # ... toasted
# Compare the results of the outer POST with the inner GET... yum
if requesting_ursula_bytes == request.data:
return Response(status=200)
else:
return Response({'error': 'Suspicious node'}, status=400)
@rest_app.route('/node_metadata', methods=["GET"])
def all_known_nodes():
headers = {'Content-Type': 'application/octet-stream'}
if this_node.known_nodes.checksum is NO_KNOWN_NODES:
return Response(b"", headers=headers, status=204)
known_nodes_bytestring = this_node.bytestring_of_known_nodes()
signature = this_node.stamp(known_nodes_bytestring)
return Response(bytes(signature) + known_nodes_bytestring,
headers=headers)
@rest_app.route('/node_metadata', methods=["POST"])
def node_metadata_exchange():
# If these nodes already have the same fleet state, no exchange is necessary.
learner_fleet_state = request.args.get('fleet')
if learner_fleet_state == this_node.known_nodes.checksum:
log.debug(
"Learner already knew fleet state {}; doing nothing.".format(
learner_fleet_state))
headers = {'Content-Type': 'application/octet-stream'}
payload = this_node.known_nodes.snapshot() + bytes(
FLEET_STATES_MATCH)
signature = this_node.stamp(payload)
return Response(bytes(signature) + payload, headers=headers)
sprouts = _node_class.batch_from_bytes(request.data)
for node in sprouts:
this_node.remember_node(node)
# TODO: What's the right status code here? 202? Different if we already knew about the node(s)?
return all_known_nodes()
@rest_app.route('/consider_arrangement', methods=['POST'])
def consider_arrangement():
from nucypher.policy.policies import Arrangement
arrangement = Arrangement.from_bytes(request.data)
# TODO: Look at the expiration and figure out if we're even staking that long. 1701
with datastore.describe(PolicyArrangement,
arrangement.id.hex(),
writeable=True) as new_policy_arrangement:
new_policy_arrangement.arrangement_id = arrangement.id.hex(
).encode()
new_policy_arrangement.expiration = arrangement.expiration
new_policy_arrangement.alice_verifying_key = arrangement.alice.stamp.as_umbral_pubkey(
)
# TODO: Fine, we'll add the arrangement here, but if we never hear from Alice again to enact it,
# we need to prune it at some point. #1700
headers = {'Content-Type': 'application/octet-stream'}
# TODO: Make this a legit response #234.
return Response(
b"This will eventually be an actual acceptance of the arrangement.",
headers=headers)
@rest_app.route("/kFrag/<id_as_hex>", methods=['POST'])
def set_policy(id_as_hex):
"""
REST endpoint for setting a kFrag.
"""
policy_message_kit = UmbralMessageKit.from_bytes(request.data)
alices_verifying_key = policy_message_kit.sender_verifying_key
alice = _alice_class.from_public_keys(
verifying_key=alices_verifying_key)
try:
cleartext = this_node.verify_from(alice,
policy_message_kit,
decrypt=True)
except InvalidSignature:
# TODO: Perhaps we log this? Essentially 355.
return Response(status_code=400)
if not this_node.federated_only:
# This splitter probably belongs somewhere canonical.
transaction_splitter = BytestringSplitter(32)
tx, kfrag_bytes = transaction_splitter(cleartext,
return_remainder=True)
try:
# Get all of the arrangements and verify that we'll be paid.
# TODO: We'd love for this part to be impossible to reduce the risk of collusion. #1274
arranged_addresses = this_node.policy_agent.fetch_arrangement_addresses_from_policy_txid(
tx, timeout=this_node.synchronous_query_timeout)
except TimeExhausted:
# Alice didn't pay. Return response with that weird status code.
this_node.suspicious_activities_witnessed['freeriders'].append(
(alice, f"No transaction matching {tx}."))
return Response(status=402)
this_node_has_been_arranged = this_node.checksum_address in arranged_addresses
if not this_node_has_been_arranged:
this_node.suspicious_activities_witnessed['freeriders'].append(
(alice,
f"The transaction {tx} does not list me as a Worker - it lists {arranged_addresses}."
))
return Response(status=402)
else:
_tx = NO_BLOCKCHAIN_CONNECTION
kfrag_bytes = cleartext
kfrag = KFrag.from_bytes(kfrag_bytes)
if not kfrag.verify(signing_pubkey=alices_verifying_key):
raise InvalidSignature("{} is invalid".format(kfrag))
with datastore.describe(PolicyArrangement, id_as_hex,
writeable=True) as policy_arrangement:
if not policy_arrangement.alice_verifying_key == alice.stamp.as_umbral_pubkey(
):
raise alice.SuspiciousActivity
policy_arrangement.kfrag = kfrag
# TODO: Sign the arrangement here. #495
return "" # TODO: Return A 200, with whatever policy metadata.
@rest_app.route('/kFrag/<id_as_hex>', methods=["DELETE"])
def revoke_arrangement(id_as_hex):
"""
REST endpoint for revoking/deleting a KFrag from a node.
"""
from nucypher.policy.collections import Revocation
revocation = Revocation.from_bytes(request.data)
log.info("Received revocation: {} -- for arrangement {}".format(
bytes(revocation).hex(), id_as_hex))
# Check that the request is the same for the provided revocation
if not id_as_hex == revocation.arrangement_id.hex():
log.debug("Couldn't identify an arrangement with id {}".format(
id_as_hex))
return Response(status_code=400)
try:
with datastore.describe(PolicyArrangement,
id_as_hex,
writeable=True) as policy_arrangement:
if revocation.verify_signature(
policy_arrangement.alice_verifying_key):
policy_arrangement.delete()
except (DatastoreTransactionError, InvalidSignature) as e:
log.debug("Exception attempting to revoke: {}".format(e))
return Response(
response='KFrag not found or revocation signature is invalid.',
status=404)
else:
log.info("KFrag successfully removed.")
return Response(response='KFrag deleted!', status=200)
@rest_app.route('/kFrag/<id_as_hex>/reencrypt', methods=["POST"])
def reencrypt_via_rest(id_as_hex):
# Get Policy Arrangement
try:
arrangement_id = binascii.unhexlify(id_as_hex)
except (binascii.Error, TypeError):
return Response(response=b'Invalid arrangement ID', status=405)
try:
# Get KFrag
# TODO: Yeah, well, what if this arrangement hasn't been enacted? 1702
with datastore.describe(PolicyArrangement,
id_as_hex) as policy_arrangement:
kfrag = policy_arrangement.kfrag
alice_verifying_key = policy_arrangement.alice_verifying_key
except RecordNotFound:
return Response(response=arrangement_id, status=404)
# Get Work Order
from nucypher.policy.collections import WorkOrder # Avoid circular import
alice_address = canonical_address_from_umbral_key(alice_verifying_key)
work_order_payload = request.data
work_order = WorkOrder.from_rest_payload(
arrangement_id=arrangement_id,
rest_payload=work_order_payload,
ursula=this_node,
alice_address=alice_address)
log.info(
f"Work Order from {work_order.bob}, signed {work_order.receipt_signature}"
)
# Re-encrypt
response = this_node._reencrypt(
kfrag=kfrag,
work_order=work_order,
alice_verifying_key=alice_verifying_key)
# Now, Ursula saves this workorder to her database...
# Note: we give the work order a random ID to store it under.
with datastore.describe(Workorder, str(uuid.uuid4()),
writeable=True) as new_workorder:
new_workorder.arrangement_id = work_order.arrangement_id
new_workorder.bob_verifying_key = work_order.bob.stamp.as_umbral_pubkey(
)
new_workorder.bob_signature = work_order.receipt_signature
headers = {'Content-Type': 'application/octet-stream'}
return Response(headers=headers, response=response)
@rest_app.route('/treasure_map/<treasure_map_id>')
def provide_treasure_map(treasure_map_id):
headers = {'Content-Type': 'application/octet-stream'}
treasure_map_index = bytes.fromhex(treasure_map_id)
try:
treasure_map = this_node.treasure_maps[treasure_map_index]
response = Response(bytes(treasure_map), headers=headers)
log.info("{} providing TreasureMap {}".format(
this_node.nickname, treasure_map_id))
except KeyError:
log.info("{} doesn't have requested TreasureMap {}".format(
this_node.stamp, treasure_map_id))
response = Response(
"No Treasure Map with ID {}".format(treasure_map_id),
status=404,
headers=headers)
return response
@rest_app.route('/treasure_map/<treasure_map_id>', methods=['POST'])
def receive_treasure_map(treasure_map_id):
# TODO: Any of the codepaths that trigger 4xx Responses here are also SuspiciousActivity.
if not this_node.federated_only:
from nucypher.policy.collections import SignedTreasureMap as _MapClass
else:
from nucypher.policy.collections import TreasureMap as _MapClass
try:
treasure_map = _MapClass.from_bytes(
bytes_representation=request.data, verify=True)
except _MapClass.InvalidSignature:
log.info("Bad TreasureMap HRAC Signature; not storing {}".format(
treasure_map_id))
return Response("This TreasureMap's HRAC is not properly signed.",
status=401)
treasure_map_index = bytes.fromhex(treasure_map_id)
# First let's see if we already have this map.
try:
previously_saved_map = this_node.treasure_maps[treasure_map_index]
except KeyError:
pass # We don't have the map. We'll validate and perhaps save it.
else:
if previously_saved_map == treasure_map:
return Response("Already have this map.", status=303)
# Otherwise, if it's a different map with the same ID, we move on to validation.
if treasure_map.public_id() == treasure_map_id:
do_store = True
else:
return Response("Can't save a TreasureMap with this ID from you.",
status=409)
if do_store and not this_node.federated_only:
alice_checksum_address = this_node.policy_agent.contract.functions.getPolicyOwner(
treasure_map._hrac[:16]).call()
do_store = treasure_map.verify_blockchain_signature(
checksum_address=alice_checksum_address)
if do_store:
log.info("{} storing TreasureMap {}".format(
this_node, treasure_map_id))
this_node.treasure_maps[treasure_map_index] = treasure_map
return Response(bytes(treasure_map), status=202)
else:
log.info(
"Bad TreasureMap ID; not storing {}".format(treasure_map_id))
return Response("This TreasureMap doesn't match a paid Policy.",
status=402)
@rest_app.route('/status/', methods=['GET'])
def status():
if request.args.get('json'):
payload = this_node.abridged_node_details()
response = jsonify(payload)
return response
else:
headers = {"Content-Type": "text/html", "charset": "utf-8"}
previous_states = list(
reversed(this_node.known_nodes.states.values()))[:5]
# Mature every known node before rendering.
for node in this_node.known_nodes:
node.mature()
try:
content = status_template.render(
this_node=this_node,
known_nodes=this_node.known_nodes,
previous_states=previous_states,
domains=serving_domains,
version=nucypher.__version__,
checksum_address=this_node.checksum_address)
except Exception as e:
log.debug("Template Rendering Exception: ".format(str(e)))
raise TemplateError(str(e)) from e
return Response(response=content, headers=headers)
return rest_app, datastore
class Policy(ABC):
"""
An edict by Alice, arranged with n Ursulas, to perform re-encryption for a specific Bob.
"""
POLICY_ID_LENGTH = 16
log = Logger("Policy")
class NotEnoughUrsulas(Exception):
"""
Raised when a Policy has been used to generate Arrangements with Ursulas insufficient number
such that we don't have enough KFrags to give to each Ursula.
"""
class EnactmentError(Exception):
"""
Raised if one or more Ursulas failed to enact the policy.
"""
def __init__(
self,
alice: Alice,
label: bytes,
expiration: maya.MayaDT,
bob: 'Bob',
kfrags: Sequence[KFrag],
public_key: UmbralPublicKey,
m: int,
):
"""
:param kfrags: A list of KFrags to distribute per this Policy.
:param label: The identity of the resource to which Bob is granted access.
"""
self.m = m
self.n = len(kfrags)
self.alice = alice
self.label = label
self.bob = bob
self.kfrags = kfrags
self.public_key = public_key
self.expiration = expiration
self._id = construct_policy_id(self.label, bytes(self.bob.stamp))
"""
# TODO: #180 - This attribute is hanging on for dear life.
After 180 is closed, it can be completely deprecated.
The "hashed resource authentication code".
A hash of:
* Alice's public key
* Bob's public key
* the label
Alice and Bob have all the information they need to construct this.
Ursula does not, so we share it with her.
"""
self.hrac = keccak_digest(
bytes(self.alice.stamp) + bytes(self.bob.stamp) +
self.label)[:HRAC_LENGTH]
def __repr__(self):
return f"{self.__class__.__name__}:{self._id.hex()[:6]}"
def _propose_arrangement(
self,
address: ChecksumAddress,
network_middleware: RestMiddleware,
) -> Tuple[Ursula, Arrangement]:
"""
Attempt to propose an arrangement to the node with the given address.
"""
if address not in self.alice.known_nodes:
raise RuntimeError(f"{address} is not known")
ursula = self.alice.known_nodes[address]
arrangement = Arrangement.from_alice(alice=self.alice,
expiration=self.expiration)
self.log.debug(f"Proposing arrangement {arrangement} to {ursula}")
negotiation_response = network_middleware.propose_arrangement(
ursula, arrangement)
status = negotiation_response.status_code
if status == 200:
self.log.debug(f"Arrangement accepted by {ursula}")
else:
message = f"Proposing arrangement to {ursula} failed with {status}"
self.log.debug(message)
raise RuntimeError(message)
# We could just return the arrangement and get the Ursula object
# from `known_nodes` later, but when we introduce slashing in FleetSensor,
# the address can already disappear from `known_nodes` by that time.
return (ursula, arrangement)
@abstractmethod
def _make_reservoir(
self, handpicked_addresses: Sequence[ChecksumAddress]
) -> MergedReservoir:
"""
Builds a `MergedReservoir` to use for drawing addresses to send proposals to.
"""
raise NotImplementedError
def _make_arrangements(
self,
network_middleware: RestMiddleware,
handpicked_ursulas: Optional[Iterable[Ursula]] = None,
timeout: int = 10,
) -> Dict[Ursula, Arrangement]:
"""
Pick some Ursula addresses and send them arrangement proposals.
Returns a dictionary of Ursulas to Arrangements if it managed to get `n` responses.
"""
if handpicked_ursulas is None:
handpicked_ursulas = []
handpicked_addresses = [
ursula.checksum_address for ursula in handpicked_ursulas
]
reservoir = self._make_reservoir(handpicked_addresses)
value_factory = PrefetchStrategy(reservoir, self.n)
def worker(address):
return self._propose_arrangement(address, network_middleware)
self.alice.block_until_number_of_known_nodes_is(
self.n, learn_on_this_thread=True, eager=True)
worker_pool = WorkerPool(worker=worker,
value_factory=value_factory,
target_successes=self.n,
timeout=timeout,
stagger_timeout=1,
threadpool_size=self.n)
worker_pool.start()
try:
successes = worker_pool.block_until_target_successes()
except (WorkerPool.OutOfValues, WorkerPool.TimedOut):
# It's possible to raise some other exceptions here,
# but we will use the logic below.
successes = worker_pool.get_successes()
finally:
worker_pool.cancel()
worker_pool.join()
accepted_arrangements = {
ursula: arrangement
for ursula, arrangement in successes.values()
}
failures = worker_pool.get_failures()
accepted_addresses = ", ".join(ursula.checksum_address
for ursula in accepted_arrangements)
if len(accepted_arrangements) < self.n:
rejected_proposals = "\n".join(
f"{address}: {exception}"
for address, exception in failures.items())
self.log.debug(
"Could not find enough Ursulas to accept proposals.\n"
f"Accepted: {accepted_addresses}\n"
f"Rejected:\n{rejected_proposals}")
raise self._not_enough_ursulas_exception()
else:
self.log.debug(
f"Finished proposing arrangements; accepted: {accepted_addresses}"
)
return accepted_arrangements
def _enact_arrangements(
self,
network_middleware: RestMiddleware,
arrangements: Dict[Ursula, Arrangement],
publication_transaction: Optional[HexBytes] = None,
publish_treasure_map: bool = True,
timeout: int = 10,
):
"""
Attempts to distribute kfrags to Ursulas that accepted arrangements earlier.
"""
def worker(ursula_and_kfrag):
ursula, kfrag = ursula_and_kfrag
arrangement = arrangements[ursula]
# TODO: seems like it would be enough to just encrypt this with Ursula's public key,
# and not create a whole capsule.
# Can't change for now since it's node protocol.
payload = self._make_enactment_payload(publication_transaction,
kfrag)
message_kit, _signature = self.alice.encrypt_for(ursula, payload)
try:
# TODO: Concurrency
response = network_middleware.enact_policy(
ursula, arrangement.id, message_kit.to_bytes())
except network_middleware.UnexpectedResponse as e:
status = e.status
else:
status = response.status_code
return status
value_factory = AllAtOnceFactory(list(zip(arrangements, self.kfrags)))
worker_pool = WorkerPool(worker=worker,
value_factory=value_factory,
target_successes=self.n,
timeout=timeout,
threadpool_size=self.n)
worker_pool.start()
# Block until everything is complete. We need all the workers to finish.
worker_pool.join()
successes = worker_pool.get_successes()
if len(successes) != self.n:
raise Policy.EnactmentError()
# TODO: Enable re-tries?
statuses = {
ursula_and_kfrag[0].checksum_address: status
for ursula_and_kfrag, status in successes.items()
}
if not all(status == 200 for status in statuses.values()):
report = "\n".join(f"{address}: {status}"
for address, status in statuses.items())
self.log.debug(
f"Policy enactment failed. Request statuses:\n{report}")
# OK, let's check: if two or more Ursulas claimed we didn't pay,
# we need to re-evaulate our situation here.
number_of_claims_of_freeloading = sum(
status == 402 for status in statuses.values())
# TODO: a better exception here?
if number_of_claims_of_freeloading > 2:
raise self.alice.NotEnoughNodes
# otherwise just raise a more generic error
raise Policy.EnactmentError()
def _make_treasure_map(
self,
network_middleware: RestMiddleware,
arrangements: Dict[Ursula, Arrangement],
) -> 'TreasureMap':
"""
Creates a treasure map for given arrangements.
"""
treasure_map = self._treasure_map_class(m=self.m)
for ursula, arrangement in arrangements.items():
treasure_map.add_arrangement(ursula, arrangement)
treasure_map.prepare_for_publication(
bob_encrypting_key=self.bob.public_keys(DecryptingPower),
bob_verifying_key=self.bob.public_keys(SigningPower),
alice_stamp=self.alice.stamp,
label=self.label)
return treasure_map
def _make_publisher(
self,
treasure_map: 'TreasureMap',
network_middleware: RestMiddleware,
) -> TreasureMapPublisher:
# TODO (#2516): remove hardcoding of 8 nodes
self.alice.block_until_number_of_known_nodes_is(
8, timeout=2, learn_on_this_thread=True)
target_nodes = self.bob.matching_nodes_among(self.alice.known_nodes)
treasure_map_bytes = bytes(
treasure_map) # prevent the closure from holding the reference
def put_treasure_map_on_node(node):
try:
response = network_middleware.put_treasure_map_on_node(
node=node, map_payload=treasure_map_bytes)
except Exception as e:
self.log.warn(f"Putting treasure map on {node} failed: {e}")
raise
if response.status_code == 201:
return response
else:
message = f"Putting treasure map on {node} failed with response status: {response.status}"
self.log.warn(message)
# TODO: What happens if this is a 300 or 400 level response?
raise Exception(message)
return TreasureMapPublisher(worker=put_treasure_map_on_node,
nodes=target_nodes)
def enact(
self,
network_middleware: RestMiddleware,
handpicked_ursulas: Optional[Iterable[Ursula]] = None,
publish_treasure_map: bool = True,
) -> 'EnactedPolicy':
"""
Attempts to enact the policy, returns an `EnactedPolicy` object on success.
"""
arrangements = self._make_arrangements(
network_middleware=network_middleware,
handpicked_ursulas=handpicked_ursulas)
self._enact_arrangements(network_middleware=network_middleware,
arrangements=arrangements,
publish_treasure_map=publish_treasure_map)
treasure_map = self._make_treasure_map(
network_middleware=network_middleware, arrangements=arrangements)
treasure_map_publisher = self._make_publisher(
treasure_map=treasure_map, network_middleware=network_middleware)
revocation_kit = RevocationKit(treasure_map, self.alice.stamp)
enacted_policy = EnactedPolicy(self._id, self.hrac, self.label,
self.public_key, treasure_map,
treasure_map_publisher, revocation_kit,
self.alice.stamp)
if publish_treasure_map is True:
enacted_policy.publish_treasure_map()
return enacted_policy
@abstractmethod
def _not_enough_ursulas_exception(self) -> Type[Exception]:
"""
Returns an exception to raise when there were not enough Ursulas
to distribute arrangements to.
"""
raise NotImplementedError
@abstractmethod
def _make_enactment_payload(self,
publication_transaction: Optional[HexBytes],
kfrag: KFrag) -> bytes:
"""
Serializes a given kfrag and policy publication transaction to send to Ursula.
"""
raise NotImplementedError
def __init__(self, json_ipc: bool, verbose: bool, quiet: bool,
no_logs: bool, console_logs: bool, file_logs: bool,
sentry_logs: bool, log_level: bool, debug: bool):
self.log = Logger(self.__class__.__name__)
# Session Emitter for pre and post character control engagement.
if verbose and quiet:
raise click.BadOptionUsage(
option_name="quiet",
message="--verbose and --quiet are mutually exclusive "
"and cannot be used at the same time.")
if verbose:
GroupGeneralConfig.verbosity = 2
elif quiet:
GroupGeneralConfig.verbosity = 0
else:
GroupGeneralConfig.verbosity = 1
if json_ipc:
GlobalLoggerSettings._json_ipc = True # TODO #1754
emitter = JSONRPCStdoutEmitter(
verbosity=GroupGeneralConfig.verbosity)
else:
emitter = StdoutEmitter(verbosity=GroupGeneralConfig.verbosity)
self.emitter = emitter
if verbose:
self.emitter.message("Verbose mode is enabled", color='blue')
# Logging
if debug and no_logs:
message = "--debug and --no-logs cannot be used at the same time."
raise click.BadOptionUsage(option_name="no-logs", message=message)
# Defaults
if file_logs is None:
file_logs = self.log_to_file
if sentry_logs is None:
sentry_logs = self.log_to_sentry
if debug:
console_logs = True
file_logs = True
sentry_logs = False
log_level = 'debug'
if no_logs:
console_logs = False
file_logs = False
sentry_logs = False
if json_ipc:
console_logs = False
GlobalLoggerSettings.set_log_level(log_level_name=log_level)
if console_logs:
GlobalLoggerSettings.start_console_logging()
if file_logs:
GlobalLoggerSettings.start_text_file_logging()
GlobalLoggerSettings.start_json_file_logging()
if sentry_logs:
GlobalLoggerSettings.start_sentry_logging(self.sentry_endpoint)
if json_ipc:
GlobalLoggerSettings.stop_console_logging() # JSON-RPC Protection
self.debug = debug
self.json_ipc = json_ipc
class RetryRequestMiddleware:
"""
Automatically retries rpc requests whenever a 429 status code is returned.
"""
def __init__(self,
make_request: Callable[[RPCEndpoint, Any], RPCResponse],
w3: Web3,
retries: int = 3,
exponential_backoff: bool = True):
self.w3 = w3
self.make_request = make_request
self.retries = retries
self.exponential_backoff = exponential_backoff
self.logger = Logger(self.__class__.__name__)
def is_request_result_retry(self, result: Union[RPCResponse,
Exception]) -> bool:
# default retry functionality - look for 429 codes
# override for additional checks
if isinstance(result, HTTPError):
# HTTPError 429
status_code = result.response.status_code
if status_code == 429:
return True
elif not isinstance(result, Exception):
# must be RPCResponse
if 'error' in result:
error = result['error']
# either instance of RPCError or str
if not isinstance(error, str) and error.get('code') == 429:
return True
# not retry result
return False
def __call__(self, method, params):
result = None
num_iterations = 1 + self.retries # initial call and subsequent retries
for i in range(num_iterations):
try:
response = self.make_request(method, params)
except Exception as e: # type: ignore
result = e
else:
result = response
# completed request
if not self.is_request_result_retry(result):
if i > 0:
# not initial call and retry was actually performed
self.logger.debug(
f'Retried rpc request completed after {i} retries')
break
# max retries with no completion
if i == self.retries:
self.logger.warn(
f'RPC request retried {self.retries} times but was not completed'
)
break
# backoff before next call
if self.exponential_backoff:
time.sleep(2**(i +
1)) # exponential back-off - 2^(retry number)
if isinstance(result, Exception):
raise result
else:
# RPCResponse
return result
class AvailabilityTracker:
FAST_INTERVAL = 15 # Seconds
SLOW_INTERVAL = 60 * 2
SEEDING_DURATION = 60
MAXIMUM_ALONE_TIME = 120
MAXIMUM_SCORE = 10.0 # Score
SAMPLE_SIZE = 1 # Ursulas
SENSITIVITY = 0.5 # Threshold
CHARGE_RATE = 0.9 # Measurement Multiplier
class Unreachable(RuntimeError):
pass
class Solitary(Unreachable):
message = "Cannot connect to any teacher nodes."
class Lonely(Unreachable):
message = "Cannot connect to enough teacher nodes."
def __init__(self, ursula, enforce_loneliness: bool = True):
self.log = Logger(self.__class__.__name__)
self._ursula = ursula
self.enforce_loneliness = enforce_loneliness
self.__excuses = dict() # List of failure reasons
self.__score = 10
# 10 == Perfect Score
self.warnings = {
9: self.mild_warning,
7: self.medium_warning,
2: self.severe_warning,
1: self.shutdown_everything # 0 is unobtainable
}
self._start_time = None
self.__active_measurement = False
self.__task = LoopingCall(self.maintain)
self.responders = set()
@property
def excuses(self):
return self.__excuses
def mild_warning(self) -> None:
self.log.info(f'[UNREACHABLE NOTICE (SCORE {self.score})] This node was recently reported as unreachable.')
def medium_warning(self) -> None:
self.log.warn(f'[UNREACHABLE CAUTION (SCORE {self.score})] This node is reporting as unreachable.'
f'Please check your network and firewall configuration.')
def severe_warning(self) -> None:
self.log.warn(f'[UNREACHABLE WARNING (SCORE {self.score})] '
f'Please check your network and firewall configuration.'
f'Auto-shutdown will commence soon if the services do not become available.')
def shutdown_everything(self, reason=None, halt_reactor=False):
self.log.warn(f'[NODE IS UNREACHABLE (SCORE {self.score})] Commencing auto-shutdown sequence...')
self._ursula.stop(halt_reactor=False)
try:
if reason:
raise reason(reason.message)
raise self.Unreachable(f'{self._ursula} is unreachable (scored {self.score}).')
finally:
if halt_reactor:
self._halt_reactor()
@staticmethod
def _halt_reactor() -> None:
if reactor.running:
reactor.stop()
def handle_measurement_errors(self, crash_on_error: bool = False, *args, **kwargs) -> None:
if args:
failure = args[0]
cleaned_traceback = failure.getTraceback().replace('{', '').replace('}', '') # FIXME: Amazing.
self.log.warn("Unhandled error during availability check: {}".format(cleaned_traceback))
if crash_on_error:
failure.raiseException()
else:
# Restart on failure
if not self.running:
self.log.debug(f"Availability check crashed, restarting...")
self.start(now=True)
def status(self) -> bool:
"""Returns current indication of availability"""
result = self.score > (self.SENSITIVITY * self.MAXIMUM_SCORE)
if not result:
for time, reason in self.__excuses.items():
self.log.info(f'[{time}] - {reason["error"]}')
return result
@property
def running(self) -> bool:
return self.__task.running
def start(self, now: bool = False):
if not self.running:
self._start_time = maya.now()
d = self.__task.start(interval=self.FAST_INTERVAL, now=now)
d.addErrback(self.handle_measurement_errors)
def stop(self) -> None:
if self.running:
self.__task.stop()
def maintain(self) -> None:
known_nodes_is_smaller_than_sample_size = len(self._ursula.known_nodes) < self.SAMPLE_SIZE
# If there are no known nodes or too few known nodes, skip this round...
# ... but not for longer than the maximum allotted alone time
if known_nodes_is_smaller_than_sample_size:
if not self._ursula.lonely and self.enforce_loneliness:
now = maya.now().epoch
delta = now - self._start_time.epoch
if delta >= self.MAXIMUM_ALONE_TIME:
self.severe_warning()
reason = self.Solitary if not self._ursula.known_nodes else self.Lonely
self.shutdown_everything(reason=reason)
return
if self.__task.interval == self.FAST_INTERVAL:
now = maya.now().epoch
delta = now - self._start_time.epoch
if delta >= self.SEEDING_DURATION:
# Slow down
self.__task.interval = self.SLOW_INTERVAL
return
if self.__active_measurement:
self.log.debug(f"Availability check already in progress - skipping this round (Score: {self.score}). ")
return # Abort
else:
self.log.debug(f"Continuing to measure availability (Score: {self.score}).")
self.__active_measurement = True
try:
self.measure_sample()
finally:
self.__active_measurement = False
delta = maya.now() - self._start_time
self.log.info(f"Current availability score is {self.score} measured since {delta}")
self.issue_warnings()
def issue_warnings(self, cascade: bool = True) -> None:
warnings = sorted(self.warnings.items(), key=lambda t: t[0])
for threshold, action in warnings:
if self.score <= threshold:
action()
if not cascade:
# Exit after the first active warning is issued
return
def sample(self, quantity: int) -> list:
population = tuple(self._ursula.known_nodes._nodes.values())
ursulas = random.sample(population=population, k=quantity)
return ursulas
@property
def score(self) -> float:
return self.__score
def record(self, result: bool = None, reason: dict = None) -> None:
"""Score the result and cache it."""
if (not result) and reason:
self.__excuses[maya.now().epoch] = reason
if result is None:
return # Actually nevermind, dont score this one...
score = int(result) + self.CHARGE_RATE * self.__score
if score >= self.MAXIMUM_SCORE:
self.__score = self.MAXIMUM_SCORE
else:
self.__score = score
self.log.debug(f"Recorded new uptime score ({self.score})")
def measure_sample(self, ursulas: list = None) -> None:
"""
Measure self-availability from a sample of Ursulas or automatically from known nodes.
Handle the possibility of unreachable or invalid remote nodes in the sample.
"""
# TODO: Relocate?
Unreachable = (*NodeSeemsToBeDown,
self._ursula.NotStaking,
self._ursula.node_storage.InvalidNodeCertificate,
self._ursula.network_middleware.UnexpectedResponse)
if not ursulas:
ursulas = self.sample(quantity=self.SAMPLE_SIZE)
for ursula_or_sprout in ursulas:
try:
self.measure(ursula_or_sprout=ursula_or_sprout)
except self._ursula.network_middleware.NotFound:
# Ignore this measurement and move on because the remote node is not compatible.
self.record(None, reason={"error": "Remote node did not support 'ping' endpoint."})
except Unreachable as e:
# This node is either not an Ursula, not available, does not support uptime checks, or is not staking...
# ...do nothing and move on without changing the score.
self.log.debug(f'{ursula_or_sprout} responded to uptime check with {e.__class__.__name__}')
continue
def measure(self, ursula_or_sprout: Union['Ursula', NodeSprout]) -> None:
"""Measure self-availability from a single remote node that participates uptime checks."""
try:
response = self._ursula.network_middleware.check_rest_availability(initiator=self._ursula, responder=ursula_or_sprout)
except RestMiddleware.BadRequest as e:
self.responders.add(ursula_or_sprout.checksum_address)
self.record(False, reason=e.reason)
else:
# Record response
self.responders.add(ursula_or_sprout.checksum_address)
if response.status_code == 200:
self.record(True)
elif response.status_code == 400:
self.record(False, reason={'failed': f"{ursula_or_sprout.checksum_address} reported unavailability."})
else:
self.record(None, reason={"error": f"{ursula_or_sprout.checksum_address} returned {response.status_code} from 'ping' endpoint."})
class Felix(Character, NucypherTokenActor):
"""
A NuCypher ERC20 faucet / Airdrop scheduler.
Felix is a web application that gives NuCypher *testnet* tokens to registered addresses
with a scheduled reduction of disbursement amounts, and an HTTP endpoint
for handling new address registration.
The main goal of Felix is to provide a source of testnet tokens for
research and the development of production-ready nucypher dApps.
"""
_default_crypto_powerups = [SigningPower]
# Intervals
DISTRIBUTION_INTERVAL = 60 # seconds
DISBURSEMENT_INTERVAL = 24 * 365 # only distribute tokens to the same address once each YEAR.
STAGING_DELAY = 10 # seconds
# Disbursement
BATCH_SIZE = 10 # transactions
MULTIPLIER = Decimal(
'0.9') # 10% reduction of previous disbursement is 0.9
# this is not relevant until the year of time declared above, passes.
MINIMUM_DISBURSEMENT = int(1e18) # NuNits (1 NU)
ETHER_AIRDROP_AMOUNT = int(1e17) # Wei (.1 ether)
MAX_INDIVIDUAL_REGISTRATIONS = 3 # Registration Limit
# Node Discovery
LEARNING_TIMEOUT = 30 # seconds
_SHORT_LEARNING_DELAY = 60 # seconds
_LONG_LEARNING_DELAY = 120 # seconds
_ROUNDS_WITHOUT_NODES_AFTER_WHICH_TO_SLOW_DOWN = 1
# Twisted
_CLOCK = reactor
_AIRDROP_QUEUE = dict()
class NoDatabase(RuntimeError):
pass
def __init__(self,
db_filepath: str,
rest_host: str,
rest_port: int,
client_password: str = None,
crash_on_error: bool = False,
distribute_ether: bool = True,
registry: BaseContractRegistry = None,
*args,
**kwargs):
# Character
super().__init__(registry=registry, *args, **kwargs)
self.log = Logger(f"felix-{self.checksum_address[-6::]}")
# Network
self.rest_port = rest_port
self.rest_host = rest_host
self.rest_app = NOT_RUNNING
self.crash_on_error = crash_on_error
# Database
self.db_filepath = db_filepath
self.db = NO_DATABASE_AVAILABLE
self.db_engine = create_engine(f'sqlite:///{self.db_filepath}',
convert_unicode=True)
# Blockchain
transacting_power = TransactingPower(password=client_password,
account=self.checksum_address,
cache=True)
self._crypto_power.consume_power_up(transacting_power)
self.token_agent = ContractAgency.get_agent(NucypherTokenAgent,
registry=registry)
self.blockchain = self.token_agent.blockchain
self.reserved_addresses = [self.checksum_address, NULL_ADDRESS]
# Update reserved addresses with deployed contracts
existing_entries = list(registry.enrolled_addresses)
self.reserved_addresses.extend(existing_entries)
# Distribution
self.__distributed = 0 # Track NU Output
self.__airdrop = 0 # Track Batch
self.__disbursement = 0 # Track Quantity
self._distribution_task = LoopingCall(f=self.airdrop_tokens)
self._distribution_task.clock = self._CLOCK
self.start_time = NOT_RUNNING
self.economics = EconomicsFactory.get_economics(registry=registry)
self.MAXIMUM_DISBURSEMENT = self.economics.maximum_allowed_locked
self.INITIAL_DISBURSEMENT = self.economics.minimum_allowed_locked * 3
# Optionally send ether with each token transaction
self.distribute_ether = distribute_ether
# Banner
self.log.info(FELIX_BANNER.format(self.checksum_address))
def __repr__(self):
class_name = self.__class__.__name__
r = f'{class_name}(checksum_address={self.checksum_address}, db_filepath={self.db_filepath})'
return r
def make_web_app(self):
from flask import request
from flask_sqlalchemy import SQLAlchemy
# WSGI/Flask Service
short_name = bytes(self.stamp).hex()[:6]
self.rest_app = Flask(f"faucet-{short_name}",
template_folder=TEMPLATES_DIR)
self.rest_app.config[
'SQLALCHEMY_DATABASE_URI'] = f'sqlite:///{self.db_filepath}'
self.rest_app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD_CONTENT_LENGTH
try:
self.rest_app.secret_key = sha256(
os.environ['NUCYPHER_FELIX_DB_SECRET'].encode()) # uses envvar
except KeyError:
raise OSError(
"The 'NUCYPHER_FELIX_DB_SECRET' is not set. Export your application secret and try again."
)
# Database
self.db = SQLAlchemy(self.rest_app)
# Database Tables
class Recipient(self.db.Model):
"""
The one and only table in Felix's database; Used to track recipients and airdrop metadata.
"""
__tablename__ = 'recipient'
id = self.db.Column(self.db.Integer, primary_key=True)
address = self.db.Column(self.db.String, nullable=False)
joined = self.db.Column(self.db.DateTime,
nullable=False,
default=datetime.utcnow)
total_received = self.db.Column(self.db.String,
default='0',
nullable=False)
last_disbursement_amount = self.db.Column(self.db.String,
nullable=False,
default=0)
last_disbursement_time = self.db.Column(self.db.DateTime,
nullable=True,
default=None)
is_staking = self.db.Column(self.db.Boolean,
nullable=False,
default=False)
def __repr__(self):
return f'{self.__class__.__name__}(id={self.id})'
self.Recipient = Recipient # Bind to outer class
# Flask decorators
rest_app = self.rest_app
#
# REST Routes
#
@rest_app.route("/status", methods=['GET'])
def status():
with ThreadedSession(self.db_engine) as session:
total_recipients = session.query(self.Recipient).count()
last_recipient = session.query(self.Recipient).filter(
self.Recipient.last_disbursement_time.isnot(
None)).order_by('last_disbursement_time').first()
last_address = last_recipient.address if last_recipient else None
last_transaction_date = last_recipient.last_disbursement_time.isoformat(
) if last_recipient else None
unfunded = session.query(self.Recipient).filter(
self.Recipient.last_disbursement_time.is_(None)).count()
return json.dumps({
"total_recipients": total_recipients,
"latest_recipient": last_address,
"latest_disburse_date": last_transaction_date,
"unfunded_recipients": unfunded,
"state": {
"eth": str(self.eth_balance),
"NU": str(self.token_balance),
"address": self.checksum_address,
"contract_address": self.token_agent.contract_address,
}
})
@rest_app.route("/register", methods=['POST'])
def register():
"""Handle new recipient registration via POST request."""
new_address = (request.form.get('address')
or request.get_json().get('address'))
if not new_address:
return Response(response="no address was supplied", status=411)
if not eth_utils.is_address(new_address):
return Response(
response=
"an invalid ethereum address was supplied. please ensure the address is a proper checksum.",
status=400)
else:
new_address = eth_utils.to_checksum_address(new_address)
if new_address in self.reserved_addresses:
return Response(
response=
"sorry, that address is reserved and cannot receive funds.",
status=403)
try:
with ThreadedSession(self.db_engine) as session:
existing = Recipient.query.filter_by(
address=new_address).all()
if len(existing) > self.MAX_INDIVIDUAL_REGISTRATIONS:
# Address already exists; Abort
self.log.debug(
f"{new_address} is already enrolled {self.MAX_INDIVIDUAL_REGISTRATIONS} times."
)
return Response(
response=
f"{new_address} requested too many times - Please use another address.",
status=409)
# Create the record
recipient = Recipient(address=new_address,
joined=datetime.now())
session.add(recipient)
session.commit()
except Exception as e:
# Pass along exceptions to the logger
self.log.critical(str(e))
raise
else:
return Response(status=200) # TODO
return rest_app
def create_tables(self) -> None:
self.make_web_app()
return self.db.create_all(app=self.rest_app)
def start(self,
host: str,
port: int,
web_services: bool = True,
distribution: bool = True,
crash_on_error: bool = False):
self.crash_on_error = crash_on_error
if self.start_time is not NOT_RUNNING:
raise RuntimeError("Felix is already running.")
self.start_time = maya.now()
payload = {"wsgi": self.rest_app, "http_port": port}
deployer = HendrixDeploy(action="start", options=payload)
if distribution is True:
self.start_distribution()
if web_services is True:
deployer.run() # <-- Blocking call (Reactor)
def start_distribution(self, now: bool = True) -> bool:
"""Start token distribution"""
self.log.info(NU_BANNER)
self.log.info("Starting NU Token Distribution | START")
if self.token_balance == NU.ZERO():
raise self.ActorError(
f"Felix address {self.checksum_address} has 0 NU tokens.")
self._distribution_task.start(interval=self.DISTRIBUTION_INTERVAL,
now=now)
return True
def stop_distribution(self) -> bool:
"""Start token distribution"""
self.log.info("Stopping NU Token Distribution | STOP")
self._distribution_task.stop()
return True
def __calculate_disbursement(self, recipient) -> int:
"""Calculate the next reward for a recipient once the are selected for distribution"""
# Initial Reward - sets the future rates
if recipient.last_disbursement_time is None:
amount = self.INITIAL_DISBURSEMENT
# Cap reached, We'll continue to leak the minimum disbursement
elif int(recipient.total_received) >= self.MAXIMUM_DISBURSEMENT:
amount = self.MINIMUM_DISBURSEMENT
# Calculate the next disbursement
else:
amount = math.ceil(
int(recipient.last_disbursement_amount) * self.MULTIPLIER)
if amount < self.MINIMUM_DISBURSEMENT:
amount = self.MINIMUM_DISBURSEMENT
return int(amount)
def __transfer(self, disbursement: int, recipient_address: str) -> str:
"""Perform a single token transfer transaction from one account to another."""
# Re-unlock from cache
self.blockchain.transacting_power.activate()
self.__disbursement += 1
receipt = self.token_agent.transfer(
amount=disbursement,
target_address=recipient_address,
sender_address=self.checksum_address)
txhash = receipt['transactionHash']
if self.distribute_ether:
ether = self.ETHER_AIRDROP_AMOUNT
transaction = {
'to': recipient_address,
'from': self.checksum_address,
'value': ether,
'gasPrice': self.blockchain.client.gas_price
}
ether_txhash = self.blockchain.client.send_transaction(transaction)
self.log.info(
f"Disbursement #{self.__disbursement} OK | NU {txhash.hex()[-6:]} | ETH {ether_txhash.hex()[:-6]} "
f"({str(NU(disbursement, 'NuNit'))} + {self.ETHER_AIRDROP_AMOUNT} wei) -> {recipient_address}"
)
else:
self.log.info(
f"Disbursement #{self.__disbursement} OK | {txhash.hex()[-6:]} |"
f"({str(NU(disbursement, 'NuNit'))} -> {recipient_address}")
return txhash
def airdrop_tokens(self):
"""
Calculate airdrop eligibility via faucet registration
and transfer tokens to selected recipients.
"""
with ThreadedSession(self.db_engine) as session:
population = session.query(self.Recipient).count()
message = f"{population} registered faucet recipients; " \
f"Distributed {str(NU(self.__distributed, 'NuNit'))} since {self.start_time.slang_time()}."
self.log.debug(message)
if population == 0:
return # Abort - no recipients are registered.
# For filtration
since = datetime.now() - timedelta(hours=self.DISBURSEMENT_INTERVAL)
datetime_filter = or_(self.Recipient.last_disbursement_time <= since,
self.Recipient.last_disbursement_time ==
None) # This must be `==` not `is`
with ThreadedSession(self.db_engine) as session:
candidates = session.query(
self.Recipient).filter(datetime_filter).all()
if not candidates:
self.log.info("No eligible recipients this round.")
return
# Discard invalid addresses, in-depth
invalid_addresses = list()
def siphon_invalid_entries(candidate):
address_is_valid = eth_utils.is_checksum_address(candidate.address)
if not address_is_valid:
invalid_addresses.append(candidate.address)
return address_is_valid
candidates = list(filter(siphon_invalid_entries, candidates))
if invalid_addresses:
self.log.info(
f"{len(invalid_addresses)} invalid entries detected. Pruning database."
)
# TODO: Is this needed? - Invalid entries are rejected at the endpoint view.
# Prune database of invalid records
# with ThreadedSession(self.db_engine) as session:
# bad_eggs = session.query(self.Recipient).filter(self.Recipient.address in invalid_addresses).all()
# for egg in bad_eggs:
# session.delete(egg.id)
# session.commit()
if not candidates:
self.log.info("No eligible recipients this round.")
return
d = threads.deferToThread(self.__do_airdrop, candidates=candidates)
self._AIRDROP_QUEUE[self.__airdrop] = d
return d
def __do_airdrop(self, candidates: list):
self.log.info(f"Staging Airdrop #{self.__airdrop}.")
# Staging
staged_disbursements = [(r, self.__calculate_disbursement(recipient=r))
for r in candidates]
batches = list(
staged_disbursements[index:index + self.BATCH_SIZE]
for index in range(0, len(staged_disbursements), self.BATCH_SIZE))
total_batches = len(batches)
self.log.info("====== Staged Airdrop ======")
for recipient, disbursement in staged_disbursements:
self.log.info(f"{recipient.address} ... {str(disbursement)[:-18]}")
self.log.info("==========================")
# Staging Delay
self.log.info(
f"Airdrop will commence in {self.STAGING_DELAY} seconds...")
if self.STAGING_DELAY > 3:
time.sleep(self.STAGING_DELAY - 3)
for i in range(3):
time.sleep(1)
self.log.info(f"NU Token airdrop starting in {3 - i} seconds...")
# Slowly, in series...
for batch, staged_disbursement in enumerate(batches, start=1):
self.log.info(f"======= Batch #{batch} ========")
for recipient, disbursement in staged_disbursement:
# Perform the transfer... leaky faucet.
self.__transfer(disbursement=disbursement,
recipient_address=recipient.address)
self.__distributed += disbursement
# Update the database record
recipient.last_disbursement_amount = str(disbursement)
recipient.total_received = str(
int(recipient.total_received) + disbursement)
recipient.last_disbursement_time = datetime.now()
self.db.session.add(recipient)
self.db.session.commit()
# end inner loop
self.log.info(
f"Completed Airdrop #{self.__airdrop} Batch #{batch} of {total_batches}."
)
# end outer loop
now = maya.now()
next_interval_slang = now.add(
seconds=self.DISTRIBUTION_INTERVAL).slang_time()
self.log.info(
f"Completed Airdrop #{self.__airdrop}; Next airdrop is {next_interval_slang}."
)
del self._AIRDROP_QUEUE[self.__airdrop]
self.__airdrop += 1
class Policy(ABC):
"""
An edict by Alice, arranged with n Ursulas, to perform re-encryption for a specific Bob
for a specific path.
Once Alice is ready to enact a Policy, she generates KFrags, which become part of the Policy.
Each Ursula is offered a Arrangement (see above) for a given Policy by Alice.
Once Alice has secured agreement with n Ursulas to enact a Policy, she sends each a KFrag,
and generates a TreasureMap for the Policy, recording which Ursulas got a KFrag.
"""
POLICY_ID_LENGTH = 16
_arrangement_class = NotImplemented
log = Logger("Policy")
class Rejected(RuntimeError):
"""Too many Ursulas rejected"""
def __init__(self,
alice: Alice,
label: bytes,
expiration: maya.MayaDT,
bob: 'Bob' = None,
kfrags: Tuple[KFrag, ...] = (UNKNOWN_KFRAG, ),
public_key=None,
m: int = None,
alice_signature=NOT_SIGNED) -> None:
"""
:param kfrags: A list of KFrags to distribute per this Policy.
:param label: The identity of the resource to which Bob is granted access.
"""
self.alice = alice
self.label = label
self.bob = bob
self.kfrags = kfrags
self.public_key = public_key
self._id = construct_policy_id(self.label, bytes(self.bob.stamp))
self.treasure_map = self._treasure_map_class(m=m)
self.expiration = expiration
self._accepted_arrangements = set() # type: Set[Arrangement]
self._rejected_arrangements = set() # type: Set[Arrangement]
self._spare_candidates = set() # type: Set[Ursula]
self._enacted_arrangements = OrderedDict()
self._published_arrangements = OrderedDict()
self.alice_signature = alice_signature # TODO: This is unused / To Be Implemented?
self.publishing_mutex = None
class MoreKFragsThanArrangements(TypeError):
"""
Raised when a Policy has been used to generate Arrangements with Ursulas insufficient number
such that we don't have enough KFrags to give to each Ursula.
"""
@property
def n(self) -> int:
return len(self.kfrags)
@property
def id(self) -> bytes:
return self._id
def __repr__(self):
return f"{self.__class__.__name__}:{self.id.hex()[:6]}"
@property
def accepted_ursulas(self) -> Set[Ursula]:
return {
arrangement.ursula
for arrangement in self._accepted_arrangements
}
def hrac(self) -> bytes:
"""
# TODO: #180 - This function is hanging on for dear life. After 180 is closed, it can be completely deprecated.
The "hashed resource authentication code".
A hash of:
* Alice's public key
* Bob's public key
* the label
Alice and Bob have all the information they need to construct this.
Ursula does not, so we share it with her.
"""
return keccak_digest(
bytes(self.alice.stamp) + bytes(self.bob.stamp) +
self.label)[:HRAC_LENGTH]
async def put_treasure_map_on_node(self, node, network_middleware):
response = network_middleware.put_treasure_map_on_node(
node=node, map_payload=bytes(self.treasure_map))
return response
def publish_treasure_map(
self,
network_middleware: RestMiddleware,
blockchain_signer: Callable = None) -> NodeEngagementMutex:
self.treasure_map.prepare_for_publication(
self.bob.public_keys(DecryptingPower),
self.bob.public_keys(SigningPower), self.alice.stamp, self.label)
if blockchain_signer is not None:
self.treasure_map.include_blockchain_signature(blockchain_signer)
self.alice.block_until_number_of_known_nodes_is(
8, timeout=2, learn_on_this_thread=True)
target_nodes = self.bob.matching_nodes_among(self.alice.known_nodes)
self.publishing_mutex = NodeEngagementMutex(
callable_to_engage=self.put_treasure_map_on_node,
nodes=target_nodes,
network_middleware=network_middleware)
self.publishing_mutex.start()
def credential(self, with_treasure_map=True):
"""
Creates a PolicyCredential for portable access to the policy via
Alice or Bob. By default, it will include the treasure_map for the
policy unless `with_treasure_map` is False.
"""
treasure_map = self.treasure_map
if not with_treasure_map:
treasure_map = None
credential = PolicyCredential(alice_verifying_key=self.alice.stamp,
label=self.label,
expiration=self.expiration,
policy_pubkey=self.public_key,
treasure_map=treasure_map)
return credential
def __assign_kfrags(self) -> Generator[Arrangement, None, None]:
if len(self._accepted_arrangements) < self.n:
raise self.MoreKFragsThanArrangements(
"Not enough candidate arrangements. "
"Call make_arrangements to make more.")
for kfrag in self.kfrags:
for arrangement in self._accepted_arrangements:
if not arrangement in self._enacted_arrangements.values():
arrangement.kfrag = kfrag
self._enacted_arrangements[kfrag] = arrangement
yield arrangement
break # This KFrag is now assigned; break the inner loop and go back to assign other kfrags.
else:
# We didn't assign that KFrag. Trouble.
# This is ideally an impossible situation, because we don't typically
# enter this method unless we've already had n or more Arrangements accepted.
raise self.MoreKFragsThanArrangements(
"Not enough accepted arrangements to assign all KFrags.")
return
def enact(self, network_middleware, publish_treasure_map=True) -> dict:
"""
Assign kfrags to ursulas_on_network, and distribute them via REST,
populating enacted_arrangements
"""
for arrangement in self.__assign_kfrags():
arrangement_message_kit = arrangement.encrypt_payload_for_ursula()
try:
# TODO: Concurrency
response = network_middleware.enact_policy(
arrangement.ursula, arrangement.id,
arrangement_message_kit.to_bytes())
except network_middleware.UnexpectedResponse as e:
arrangement.status = e.status
else:
arrangement.status = response.status_code
# TODO: Handle problem here - if the arrangement is bad, deal with it.
self.treasure_map.add_arrangement(arrangement)
else:
# OK, let's check: if two or more Ursulas claimed we didn't pay,
# we need to re-evaulate our situation here.
arrangement_statuses = [
a.status for a in self._accepted_arrangements
]
number_of_claims_of_freeloading = sum(
status == 402 for status in arrangement_statuses)
if number_of_claims_of_freeloading > 2:
raise self.alice.NotEnoughNodes # TODO: Clean this up and enable re-tries.
self.treasure_map.check_for_sufficient_destinations()
# TODO: Leave a note to try any failures later.
pass
# ...After *all* the arrangements are enacted
# Create Alice's revocation kit
self.revocation_kit = RevocationKit(self, self.alice.stamp)
self.alice.add_active_policy(self)
if publish_treasure_map is True:
return self.publish_treasure_map(
network_middleware=network_middleware
) # TODO: blockchain_signer?
def propose_arrangement(self, network_middleware, arrangement) -> bool:
negotiation_response = network_middleware.propose_arrangement(
arrangement=arrangement) # Wow, we aren't even passing node here.
# TODO: check out the response: need to assess the result and see if we're actually good to go.
arrangement_is_accepted = negotiation_response.status_code == 200
bucket = self._accepted_arrangements if arrangement_is_accepted else self._rejected_arrangements
bucket.add(arrangement)
return arrangement_is_accepted
def make_arrangements(
self,
network_middleware: RestMiddleware,
handpicked_ursulas: Optional[Set[Ursula]] = None,
discover_on_this_thread: bool = True,
*args,
**kwargs,
) -> None:
sampled_ursulas = self.sample(
handpicked_ursulas=handpicked_ursulas,
discover_on_this_thread=discover_on_this_thread)
if len(sampled_ursulas) < self.n:
raise self.MoreKFragsThanArrangements(
"To make a Policy in federated mode, you need to designate *all* ' \
the Ursulas you need (in this case, {}); there's no other way to ' \
know which nodes to use. Either pass them here or when you make ' \
the Policy.".format(self.n))
# TODO: One of these layers needs to add concurrency.
self._propose_arrangements(network_middleware=network_middleware,
candidate_ursulas=sampled_ursulas,
*args,
**kwargs)
if len(self._accepted_arrangements) < self.n:
formatted_offenders = '\n'.join(
f'{u.checksum_address}@{u.rest_url()}'
for u in sampled_ursulas)
raise self.Rejected(
f'Selected Ursulas rejected too many arrangements'
f'- only {len(self._accepted_arrangements)} of {self.n} accepted.\n'
f'Offending nodes: \n{formatted_offenders}\n')
@abstractmethod
def make_arrangement(self, ursula: Ursula, *args, **kwargs):
raise NotImplementedError
@abstractmethod
def sample_essential(self, *args, **kwargs) -> Set[Ursula]:
raise NotImplementedError
def sample(
self,
handpicked_ursulas: Optional[Set[Ursula]] = None,
discover_on_this_thread: bool = False,
) -> Set[Ursula]:
selected_ursulas = set(
handpicked_ursulas) if handpicked_ursulas else set()
# Calculate the target sample quantity
if self.n - len(selected_ursulas) > 0:
sampled_ursulas = self.sample_essential(
handpicked_ursulas=selected_ursulas,
discover_on_this_thread=discover_on_this_thread)
selected_ursulas.update(sampled_ursulas)
return selected_ursulas
def _propose_arrangements(self,
network_middleware: RestMiddleware,
candidate_ursulas: Set[Ursula],
consider_everyone: bool = False,
*args,
**kwargs) -> None:
for index, selected_ursula in enumerate(candidate_ursulas):
arrangement = self.make_arrangement(ursula=selected_ursula,
*args,
**kwargs)
try:
is_accepted = self.propose_arrangement(
arrangement=arrangement,
network_middleware=network_middleware)
except NodeSeemsToBeDown as e: # TODO: #355 Also catch InvalidNode here?
# This arrangement won't be added to the accepted bucket.
# If too many nodes are down, it will fail in make_arrangements.
# Also TODO: Prolly log this or something at this stage.
continue
else:
# Bucket the arrangements
if is_accepted:
self.log.debug(
f"Arrangement accepted by {selected_ursula}")
self._accepted_arrangements.add(arrangement)
accepted = len(self._accepted_arrangements)
if accepted == self.n and not consider_everyone:
try:
spares = set(list(candidate_ursulas)[index + 1::])
self._spare_candidates.update(spares)
except IndexError:
self._spare_candidates = set()
break
else:
self.log.debug(
f"Arrangement failed with {selected_ursula}")
self._rejected_arrangements.add(arrangement)
from nucypher.characters.control.emitters import JSONRPCStdoutEmitter
from nucypher.characters.lawful import Ursula
from nucypher.cli import utils
from nucypher.cli.literature import SUCCESSFUL_DESTRUCTION
from nucypher.cli.main import nucypher_cli
from nucypher.config.characters import BobConfiguration
from nucypher.config.constants import TEMPORARY_DOMAIN
from nucypher.crypto.kits import UmbralMessageKit
from nucypher.crypto.powers import SigningPower
from nucypher.utilities.logging import GlobalLoggerSettings, Logger
from tests.constants import (FAKE_PASSWORD_CONFIRMED,
INSECURE_DEVELOPMENT_PASSWORD,
MOCK_CUSTOM_INSTALLATION_PATH)
log = Logger()
@mock.patch('nucypher.config.characters.BobConfiguration.default_filepath',
return_value='/non/existent/file')
def test_missing_configuration_file(default_filepath_mock, click_runner):
cmd_args = ('bob', 'run')
result = click_runner.invoke(nucypher_cli,
cmd_args,
catch_exceptions=False)
assert result.exit_code != 0
assert default_filepath_mock.called
assert "nucypher bob init" in result.output
def test_bob_public_keys(click_runner):
class FleetSensor:
"""
A representation of a fleet of NuCypher nodes.
"""
_checksum = NO_KNOWN_NODES.bool_value(False)
_nickname = NO_KNOWN_NODES
_tracking = False
most_recent_node_change = NO_KNOWN_NODES
snapshot_splitter = BytestringSplitter(32, 4)
log = Logger("Learning")
FleetState = namedtuple(
"FleetState", ("nickname", "icon", "nodes", "updated", "checksum"))
def __init__(self, domain: str):
self.domain = domain
self.additional_nodes_to_track = []
self.updated = maya.now()
self._nodes = OrderedDict()
self._marked = defaultdict(list) # Beginning of bucketing.
self.states = OrderedDict()
def __setitem__(self, checksum_address, node_or_sprout):
if node_or_sprout.domain == self.domain:
self._nodes[checksum_address] = node_or_sprout
if self._tracking:
self.log.info(
"Updating fleet state after saving node {}".format(
node_or_sprout))
self.record_fleet_state()
else:
msg = f"Rejected node {node_or_sprout} because its domain is '{node_or_sprout.domain}' but we're only tracking '{self.domain}'"
self.log.warn(msg)
def __getitem__(self, checksum_address):
return self._nodes[checksum_address]
def __bool__(self):
return bool(self._nodes)
def __contains__(self, item):
return item in self._nodes.keys() or item in self._nodes.values()
def __iter__(self):
yield from self._nodes.values()
def __len__(self):
return len(self._nodes)
def __eq__(self, other):
return self._nodes == other._nodes
def __repr__(self):
return self._nodes.__repr__()
def population(self):
return len(self) + len(self.additional_nodes_to_track)
@property
def checksum(self):
return self._checksum
@checksum.setter
def checksum(self, checksum_value):
self._checksum = checksum_value
self._nickname = Nickname.from_seed(checksum_value, length=1)
@property
def nickname(self):
return self._nickname
@property
def icon(self) -> str:
if self.nickname is NO_KNOWN_NODES:
return str(NO_KNOWN_NODES)
return self.nickname.icon
def addresses(self):
return self._nodes.keys()
def snapshot(self):
fleet_state_checksum_bytes = binascii.unhexlify(self.checksum)
fleet_state_updated_bytes = self.updated.epoch.to_bytes(
4, byteorder="big")
return fleet_state_checksum_bytes + fleet_state_updated_bytes
def record_fleet_state(self, additional_nodes_to_track=None):
if additional_nodes_to_track:
self.additional_nodes_to_track.extend(additional_nodes_to_track)
if not self._nodes:
# No news here.
return
sorted_nodes = self.sorted()
sorted_nodes_joined = b"".join(bytes(n) for n in sorted_nodes)
checksum = keccak_digest(sorted_nodes_joined).hex()
if checksum not in self.states:
self.checksum = keccak_digest(b"".join(
bytes(n) for n in self.sorted())).hex()
self.updated = maya.now()
# For now we store the sorted node list. Someday we probably spin this out into
# its own class, FleetState, and use it as the basis for partial updates.
new_state = self.FleetState(nickname=self.nickname,
nodes=sorted_nodes,
icon=self.icon,
updated=self.updated,
checksum=self.checksum)
self.states[checksum] = new_state
return checksum, new_state
def start_tracking_state(self, additional_nodes_to_track=None):
if additional_nodes_to_track is None:
additional_nodes_to_track = list()
self.additional_nodes_to_track.extend(additional_nodes_to_track)
self._tracking = True
self.update_fleet_state()
def sorted(self):
nodes_to_consider = list(
self._nodes.values()) + self.additional_nodes_to_track
return sorted(nodes_to_consider, key=lambda n: n.checksum_address)
def shuffled(self):
nodes_we_know_about = list(self._nodes.values())
random.shuffle(nodes_we_know_about)
return nodes_we_know_about
def abridged_states_dict(self):
abridged_states = {}
for k, v in self.states.items():
abridged_states[k] = self.abridged_state_details(v)
return abridged_states
@staticmethod
def abridged_state_details(state):
return {
"nickname": str(state.nickname),
# FIXME: generalize in case we want to extend the number of symbols in the state nickname
"symbol": state.nickname.characters[0].symbol,
"color_hex": state.nickname.characters[0].color_hex,
"color_name": state.nickname.characters[0].color_name,
"updated": state.updated.rfc2822(),
}
def mark_as(self, label: Exception, node: "Teacher"):
self._marked[label].append(node)
if self._nodes.get(node):
del self._nodes[node]
def __init__(
self,
# Base
emitter=None,
config_root: str = None,
filepath: str = None,
# Mode
dev_mode: bool = False,
federated_only: bool = False,
# Identity
checksum_address: str = None,
crypto_power: CryptoPower = None,
# Keyring
keyring: NucypherKeyring = None,
keyring_root: str = None,
# Learner
learn_on_same_thread: bool = False,
abort_on_learning_error: bool = False,
start_learning_now: bool = True,
# Network
controller_port: int = None,
domain: str = DEFAULT_DOMAIN,
interface_signature: Signature = None,
network_middleware: RestMiddleware = None,
lonely: bool = False,
# Node Storage
known_nodes: set = None,
node_storage: NodeStorage = None,
reload_metadata: bool = True,
save_metadata: bool = True,
# Blockchain
poa: bool = None,
light: bool = False,
sync: bool = False,
provider_uri: str = None,
provider_process=None,
gas_strategy: Union[Callable, str] = DEFAULT_GAS_STRATEGY,
signer_uri: str = None,
# Registry
registry: BaseContractRegistry = None,
registry_filepath: str = None,
# Deployed Workers
worker_data: dict = None):
self.log = Logger(self.__class__.__name__)
UNINITIALIZED_CONFIGURATION.bool_value(False)
# Identity
# NOTE: NodeConfigurations can only be used with Self-Characters
self.is_me = True
self.checksum_address = checksum_address
# Keyring
self.crypto_power = crypto_power
self.keyring = keyring or NO_KEYRING_ATTACHED
self.keyring_root = keyring_root or UNINITIALIZED_CONFIGURATION
# Contract Registry
if registry and registry_filepath:
if registry.filepath != registry_filepath:
error = f"Inconsistent registry filepaths for '{registry.filepath}' and '{registry_filepath}'."
raise ValueError(error)
else:
self.log.warn(
f"Registry and registry filepath were both passed.")
self.registry = registry or NO_BLOCKCHAIN_CONNECTION.bool_value(False)
self.registry_filepath = registry_filepath or UNINITIALIZED_CONFIGURATION
# Blockchain
self.poa = poa
self.is_light = light
self.provider_uri = provider_uri or NO_BLOCKCHAIN_CONNECTION
self.provider_process = provider_process or NO_BLOCKCHAIN_CONNECTION
self.signer_uri = signer_uri or None
# Learner
self.federated_only = federated_only
self.domain = domain
self.learn_on_same_thread = learn_on_same_thread
self.abort_on_learning_error = abort_on_learning_error
self.start_learning_now = start_learning_now
self.save_metadata = save_metadata
self.reload_metadata = reload_metadata
self.known_nodes = known_nodes or set() # handpicked
self.lonely = lonely
# Configuration
self.__dev_mode = dev_mode
self.config_file_location = filepath or UNINITIALIZED_CONFIGURATION
self.config_root = UNINITIALIZED_CONFIGURATION
# Deployed Workers
self.worker_data = worker_data
#
# Federated vs. Blockchain arguments consistency
#
#
# Federated
#
if self.federated_only:
# Check for incompatible values
blockchain_args = {
'filepath': registry_filepath,
'poa': poa,
'provider_process': provider_process,
'provider_uri': provider_uri,
'gas_strategy': gas_strategy
}
if any(blockchain_args.values()):
bad_args = ", ".join(f"{arg}={val}"
for arg, val in blockchain_args.items()
if val)
self.log.warn(
f"Arguments {bad_args} are incompatible with federated_only. "
f"Overridden with a sane default.")
# Clear decentralized attributes to ensure consistency with a
# federated configuration.
self.poa = False
self.is_light = False
self.provider_uri = None
self.provider_process = None
self.registry_filepath = None
self.gas_strategy = None
#
# Decentralized
#
else:
self.gas_strategy = gas_strategy
is_initialized = BlockchainInterfaceFactory.is_interface_initialized(
provider_uri=self.provider_uri)
if not is_initialized and provider_uri:
BlockchainInterfaceFactory.initialize_interface(
provider_uri=self.provider_uri,
poa=self.poa,
light=self.is_light,
provider_process=self.provider_process,
sync=sync,
emitter=emitter,
gas_strategy=gas_strategy)
else:
self.log.warn(
f"Using existing blockchain interface connection ({self.provider_uri})."
)
if not self.registry:
# TODO: These two code blocks are untested.
if not self.registry_filepath: # TODO: Registry URI (goerli://speedynet.json) :-)
self.log.info(f"Fetching latest registry from source.")
self.registry = InMemoryContractRegistry.from_latest_publication(
network=self.domain)
else:
self.registry = LocalContractRegistry(
filepath=self.registry_filepath)
self.log.info(f"Using local registry ({self.registry}).")
if dev_mode:
self.__temp_dir = UNINITIALIZED_CONFIGURATION
self.__setup_node_storage()
self.initialize(password=DEVELOPMENT_CONFIGURATION)
else:
self.__temp_dir = LIVE_CONFIGURATION
self.config_root = config_root or self.DEFAULT_CONFIG_ROOT
self._cache_runtime_filepaths()
self.__setup_node_storage(node_storage=node_storage)
# Network
self.controller_port = controller_port or self.DEFAULT_CONTROLLER_PORT
self.network_middleware = network_middleware or self.DEFAULT_NETWORK_MIDDLEWARE(
registry=self.registry)
self.interface_signature = interface_signature
super().__init__(filepath=self.config_file_location,
config_root=self.config_root)
class TesterBlockchain(BlockchainDeployerInterface):
"""
Blockchain subclass with additional test utility methods and options.
"""
__test__ = False # prohibit pytest from collecting this object as a test
# Solidity
SOURCES: List[SourceBundle] = [
SourceBundle(base_path=SOLIDITY_SOURCE_ROOT,
other_paths=(TEST_SOLIDITY_SOURCE_ROOT, ))
]
# Web3
GAS_STRATEGIES = {
**BlockchainDeployerInterface.GAS_STRATEGIES, 'free':
free_gas_price_strategy
}
PROVIDER_URI = PYEVM_DEV_URI
DEFAULT_GAS_STRATEGY = 'free'
# Reserved addresses
_ETHERBASE = 0
_ALICE = 1
_BOB = 2
_FIRST_STAKER = 5
_FIRST_URSULA = _FIRST_STAKER + NUMBER_OF_STAKERS_IN_BLOCKCHAIN_TESTS
# Internal
__STAKERS_RANGE = range(NUMBER_OF_STAKERS_IN_BLOCKCHAIN_TESTS)
__WORKERS_RANGE = range(NUMBER_OF_URSULAS_IN_BLOCKCHAIN_TESTS)
__ACCOUNT_CACHE = list()
# Defaults
DEFAULT_ECONOMICS = StandardTokenEconomics()
def __init__(self,
test_accounts: int = NUMBER_OF_ETH_TEST_ACCOUNTS,
poa: bool = True,
light: bool = False,
eth_airdrop: bool = False,
free_transactions: bool = False,
*args,
**kwargs):
self.free_transactions = free_transactions
EXPECTED_CONFIRMATION_TIME_IN_SECONDS[
'free'] = 5 # Just some upper-limit
super().__init__(provider_uri=self.PROVIDER_URI,
provider_process=None,
poa=poa,
light=light,
*args,
**kwargs)
self.log = Logger("test-blockchain")
self.connect()
# Generate additional ethereum accounts for testing
population = test_accounts
enough_accounts = len(self.client.accounts) >= population
if not enough_accounts:
accounts_to_make = population - len(self.client.accounts)
self.__generate_insecure_unlocked_accounts(
quantity=accounts_to_make)
assert test_accounts == len(self.w3.eth.accounts)
if eth_airdrop is True: # ETH for everyone!
self.ether_airdrop(amount=DEVELOPMENT_ETH_AIRDROP_AMOUNT)
def attach_middleware(self):
if self.free_transactions:
self.w3.eth.setGasPriceStrategy(free_gas_price_strategy)
def __generate_insecure_unlocked_accounts(self,
quantity: int) -> List[str]:
#
# Sanity Check - Only PyEVM can be used.
#
# Detect provider platform
client_version = self.w3.clientVersion
if 'Geth' in client_version:
raise RuntimeError("WARNING: Geth providers are not implemented.")
elif "Parity" in client_version:
raise RuntimeError(
"WARNING: Parity providers are not implemented.")
addresses = list()
for _ in range(quantity):
address = self.provider.ethereum_tester.add_account(
'0x' + os.urandom(32).hex())
addresses.append(address)
self.__ACCOUNT_CACHE.append(address)
self.log.info('Generated new insecure account {}'.format(address))
return addresses
def ether_airdrop(self, amount: int) -> List[str]:
"""Airdrops ether from creator address to all other addresses!"""
coinbase, *addresses = self.w3.eth.accounts
tx_hashes = list()
for address in addresses:
tx = {'to': address, 'from': coinbase, 'value': amount}
txhash = self.w3.eth.sendTransaction(tx)
_receipt = self.wait_for_receipt(txhash)
tx_hashes.append(txhash)
eth_amount = Web3().fromWei(amount, 'ether')
self.log.info("Airdropped {} ETH {} -> {}".format(
eth_amount, tx['from'], tx['to']))
return tx_hashes
def time_travel(self,
hours: int = None,
seconds: int = None,
periods: int = None):
"""
Wait the specified number of wait_hours by comparing
block timestamps and mines a single block.
"""
more_than_one_arg = sum(map(bool, (hours, seconds, periods))) > 1
if more_than_one_arg:
raise ValueError(
"Specify hours, seconds, or periods, not a combination")
if periods:
duration = self.DEFAULT_ECONOMICS.seconds_per_period * periods
base = self.DEFAULT_ECONOMICS.seconds_per_period
elif hours:
duration = hours * (60 * 60)
base = 60 * 60
elif seconds:
duration = seconds
base = 1
else:
raise ValueError("Specify either hours, seconds, or periods.")
now = self.w3.eth.getBlock('latest').timestamp
end_timestamp = ((now + duration) // base) * base
self.w3.eth.web3.testing.timeTravel(timestamp=end_timestamp)
self.w3.eth.web3.testing.mine(1)
delta = maya.timedelta(seconds=end_timestamp - now)
self.log.info(
f"Time traveled {delta} "
f"| period {epoch_to_period(epoch=end_timestamp, seconds_per_period=self.DEFAULT_ECONOMICS.seconds_per_period)} "
f"| epoch {end_timestamp}")
@classmethod
def bootstrap_network(
cls,
economics: BaseEconomics = None
) -> Tuple['TesterBlockchain', 'InMemoryContractRegistry']:
"""For use with metric testing scripts"""
registry = InMemoryContractRegistry()
testerchain = cls()
BlockchainInterfaceFactory.register_interface(testerchain)
power = TransactingPower(password=INSECURE_DEVELOPMENT_PASSWORD,
account=testerchain.etherbase_account)
power.activate()
testerchain.transacting_power = power
origin = testerchain.client.etherbase
deployer = ContractAdministrator(deployer_address=origin,
registry=registry,
economics=economics
or cls.DEFAULT_ECONOMICS,
staking_escrow_test_mode=True)
_receipts = deployer.deploy_network_contracts(interactive=False)
return testerchain, registry
@property
def etherbase_account(self):
return self.client.accounts[self._ETHERBASE]
@property
def alice_account(self):
return self.client.accounts[self._ALICE]
@property
def bob_account(self):
return self.client.accounts[self._BOB]
def ursula_account(self, index):
if index not in self.__WORKERS_RANGE:
raise ValueError(
f"Ursula index must be lower than {NUMBER_OF_URSULAS_IN_BLOCKCHAIN_TESTS}"
)
return self.client.accounts[index + self._FIRST_URSULA]
def staker_account(self, index):
if index not in self.__STAKERS_RANGE:
raise ValueError(
f"Staker index must be lower than {NUMBER_OF_STAKERS_IN_BLOCKCHAIN_TESTS}"
)
return self.client.accounts[index + self._FIRST_STAKER]
@property
def ursulas_accounts(self):
return list(self.ursula_account(i) for i in self.__WORKERS_RANGE)
@property
def stakers_accounts(self):
return list(self.staker_account(i) for i in self.__STAKERS_RANGE)
@property
def unassigned_accounts(self):
special_accounts = [
self.etherbase_account, self.alice_account, self.bob_account
]
assigned_accounts = set(self.stakers_accounts + self.ursulas_accounts +
special_accounts)
accounts = set(self.client.accounts)
return list(accounts.difference(assigned_accounts))
def wait_for_receipt(self,
txhash: Union[bytes, str, HexBytes],
timeout: int = None) -> dict:
"""Wait for a transaction receipt and return it"""
timeout = timeout or self.TIMEOUT
result = self.client.wait_for_receipt(transaction_hash=txhash,
timeout=timeout)
if result.status == 0:
raise TransactionFailed()
return result
def get_block_number(self) -> int:
return self.client.block_number
def read_storage_slot(self, address, slot):
# https://github.com/ethereum/web3.py/issues/1490
address = to_canonical_address(address)
return self.client.w3.provider.ethereum_tester.backend.chain.get_vm(
).state.get_storage(address, slot)
class RestMiddleware:
log = Logger()
_client_class = NucypherMiddlewareClient
class Unreachable(Exception):
def __init__(self, message, *args, **kwargs):
super().__init__(message, *args, **kwargs)
class UnexpectedResponse(Exception):
def __init__(self, message, status, *args, **kwargs):
super().__init__(message, *args, **kwargs)
self.status = status
class NotFound(UnexpectedResponse):
def __init__(self, *args, **kwargs):
super().__init__(status=HTTPStatus.NOT_FOUND, *args, **kwargs)
class BadRequest(UnexpectedResponse):
def __init__(self, reason, *args, **kwargs):
self.reason = reason
super().__init__(message=reason,
status=HTTPStatus.BAD_REQUEST,
*args,
**kwargs)
class PaymentRequired(UnexpectedResponse):
"""Raised for HTTP 402"""
def __init__(self, *args, **kwargs):
super().__init__(status=HTTPStatus.PAYMENT_REQUIRED,
*args,
**kwargs)
class Unauthorized(UnexpectedResponse):
"""Raised for HTTP 403"""
def __init__(self, *args, **kwargs):
super().__init__(status=HTTPStatus.FORBIDDEN, *args, **kwargs)
def __init__(self, registry=None, eth_provider_uri: str = None):
self.client = self._client_class(registry=registry,
eth_provider_uri=eth_provider_uri)
def request_revocation(self, ursula, revocation):
# TODO: Implement offchain revocation #2787
response = self.client.post(
node_or_sprout=ursula,
path=f"revoke",
data=bytes(revocation),
)
return response
def reencrypt(self, ursula: 'Ursula', reencryption_request_bytes: bytes):
response = self.client.post(node_or_sprout=ursula,
path=f"reencrypt",
data=reencryption_request_bytes,
timeout=2)
return response
def check_availability(self, initiator, responder):
response = self.client.post(
node_or_sprout=responder,
data=bytes(initiator.metatada()),
path="check_availability",
timeout=6, # Two round trips are expected
)
return response
def ping(self, node):
response = self.client.get(node_or_sprout=node, path="ping", timeout=2)
return response
def get_nodes_via_rest(self, node,
fleet_state_checksum: FleetStateChecksum,
announce_nodes: Sequence[NodeMetadata]):
request = MetadataRequest(fleet_state_checksum=fleet_state_checksum,
announce_nodes=announce_nodes)
response = self.client.post(
node_or_sprout=node,
path="node_metadata",
data=bytes(request),
)
return response
def test_collect_rewards_integration(
click_runner, testerchain, agency_local_registry,
stakeholder_configuration_file_location, blockchain_alice,
blockchain_bob, random_policy_label, beneficiary,
preallocation_escrow_agent, mock_allocation_registry, manual_worker,
token_economics, mock_transacting_power_activation, stake_value,
policy_value, policy_rate):
# Disable re-staking
restake_args = ('stake', 'restake', '--disable', '--config-file',
stakeholder_configuration_file_location,
'--allocation-filepath',
MOCK_INDIVIDUAL_ALLOCATION_FILEPATH, '--force')
result = click_runner.invoke(nucypher_cli,
restake_args,
input=INSECURE_DEVELOPMENT_PASSWORD,
catch_exceptions=False)
assert result.exit_code == 0
half_stake_time = token_economics.minimum_locked_periods // 2 # Test setup
logger = Logger("Test-CLI") # Enter the Teacher's Logger, and
current_period = 0 # State the initial period for incrementing
staker_address = preallocation_escrow_agent.principal_contract.address
worker_address = manual_worker
# The staker is staking.
stakes = StakeList(registry=agency_local_registry,
checksum_address=staker_address)
stakes.refresh()
assert stakes
staking_agent = ContractAgency.get_agent(StakingEscrowAgent,
registry=agency_local_registry)
assert worker_address == staking_agent.get_worker_from_staker(
staker_address=staker_address)
ursula_port = select_test_port()
ursula = Ursula(is_me=True,
checksum_address=staker_address,
worker_address=worker_address,
registry=agency_local_registry,
rest_host='127.0.0.1',
rest_port=ursula_port,
start_working_now=False,
network_middleware=MockRestMiddleware(),
db_filepath=tempfile.mkdtemp())
MOCK_KNOWN_URSULAS_CACHE[ursula_port] = ursula
assert ursula.worker_address == worker_address
assert ursula.checksum_address == staker_address
mock_transacting_power_activation(account=worker_address,
password=INSECURE_DEVELOPMENT_PASSWORD)
# Make a commitment for half the first stake duration
for _ in range(half_stake_time):
logger.debug(
f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<")
ursula.commit_to_next_period()
testerchain.time_travel(periods=1)
current_period += 1
# Alice creates a policy and grants Bob access
blockchain_alice.selection_buffer = 1
M, N = 1, 1
days = 3
now = testerchain.w3.eth.getBlock(block_identifier='latest').timestamp
expiration = maya.MayaDT(now).add(days=days - 1)
blockchain_policy = blockchain_alice.grant(bob=blockchain_bob,
label=random_policy_label,
m=M,
n=N,
value=policy_value,
expiration=expiration,
handpicked_ursulas={ursula})
# Ensure that the handpicked Ursula was selected for the policy
arrangement = list(blockchain_policy._accepted_arrangements)[0]
assert arrangement.ursula == ursula
# Bob learns about the new staker and joins the policy
blockchain_bob.start_learning_loop()
blockchain_bob.remember_node(node=ursula)
blockchain_bob.join_policy(random_policy_label,
bytes(blockchain_alice.stamp))
# Enrico Encrypts (of course)
enrico = Enrico(policy_encrypting_key=blockchain_policy.public_key,
network_middleware=MockRestMiddleware())
verifying_key = blockchain_alice.stamp.as_umbral_pubkey()
for index in range(half_stake_time - 5):
logger.debug(
f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<")
ursula.commit_to_next_period()
# Encrypt
random_data = os.urandom(random.randrange(20, 100))
message_kit, signature = enrico.encrypt_message(plaintext=random_data)
# Decrypt
cleartexts = blockchain_bob.retrieve(message_kit,
enrico=enrico,
alice_verifying_key=verifying_key,
label=random_policy_label)
assert random_data == cleartexts[0]
# Ursula Staying online and the clock advancing
testerchain.time_travel(periods=1)
current_period += 1
# Finish the passage of time
for _ in range(
5 - 1
): # minus 1 because the first period was already committed to in test_ursula_run
logger.debug(
f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<")
ursula.commit_to_next_period()
current_period += 1
testerchain.time_travel(periods=1)
#
# WHERES THE MONEY URSULA?? - Collecting Rewards
#
balance = testerchain.client.get_balance(beneficiary)
# Rewards will be unlocked after the
# final committed period has passed (+1).
logger.debug(f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<")
testerchain.time_travel(periods=1)
current_period += 1
logger.debug(f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<")
# Since we are mocking the blockchain connection, manually consume the transacting power of the Beneficiary.
mock_transacting_power_activation(account=beneficiary,
password=INSECURE_DEVELOPMENT_PASSWORD)
# Collect Policy Fee
collection_args = ('stake', 'collect-reward', '--config-file',
stakeholder_configuration_file_location, '--policy-fee',
'--no-staking-reward', '--withdraw-address',
beneficiary, '--allocation-filepath',
MOCK_INDIVIDUAL_ALLOCATION_FILEPATH, '--force')
result = click_runner.invoke(nucypher_cli,
collection_args,
input=INSECURE_DEVELOPMENT_PASSWORD,
catch_exceptions=False)
assert result.exit_code == 0
# Policy Fee
collected_policy_fee = testerchain.client.get_balance(beneficiary)
assert collected_policy_fee > balance
#
# Collect Staking Reward
#
token_agent = ContractAgency.get_agent(agent_class=NucypherTokenAgent,
registry=agency_local_registry)
balance_before_collecting = token_agent.get_balance(address=staker_address)
collection_args = ('stake', 'collect-reward', '--config-file',
stakeholder_configuration_file_location,
'--no-policy-fee', '--staking-reward',
'--allocation-filepath',
MOCK_INDIVIDUAL_ALLOCATION_FILEPATH, '--force')
result = click_runner.invoke(nucypher_cli,
collection_args,
input=INSECURE_DEVELOPMENT_PASSWORD,
catch_exceptions=False)
assert result.exit_code == 0
# The beneficiary has withdrawn her staking rewards, which are now in the staking contract
assert token_agent.get_balance(
address=staker_address) >= balance_before_collecting
class BlockchainInterface:
"""
Interacts with a solidity compiler and a registry in order to instantiate compiled
ethereum contracts with the given web3 provider backend.
"""
TIMEOUT = 600 # seconds # TODO: Correlate with the gas strategy - #2070
DEFAULT_GAS_STRATEGY = 'fast'
GAS_STRATEGIES = {'glacial': time_based.glacial_gas_price_strategy, # 24h
'slow': time_based.slow_gas_price_strategy, # 1h
'medium': time_based.medium_gas_price_strategy, # 5m
'fast': time_based.fast_gas_price_strategy # 60s
}
process = NO_PROVIDER_PROCESS.bool_value(False)
Web3 = Web3
_contract_factory = VersionedContract
class InterfaceError(Exception):
pass
class NoProvider(InterfaceError):
pass
class UnsupportedProvider(InterfaceError):
pass
class ConnectionFailed(InterfaceError):
pass
class UnknownContract(InterfaceError):
pass
REASONS = {
INSUFFICIENT_ETH: 'insufficient funds for gas * price + value',
}
class TransactionFailed(InterfaceError):
IPC_CODE = -32000 # (geth)
def __init__(self,
message: str,
transaction_dict: dict,
contract_function: Union[ContractFunction, ContractConstructor],
*args):
self.base_message = message
self.name = get_transaction_name(contract_function=contract_function)
self.payload = transaction_dict
self.contract_function = contract_function
self.failures = {
BlockchainInterface.REASONS[INSUFFICIENT_ETH]: self.insufficient_eth
}
self.message = self.failures.get(self.base_message, self.default)
super().__init__(self.message, *args)
@property
def default(self) -> str:
sender = self.payload["from"]
message = f'{self.name} from {sender[:6]}... \n' \
f'Sender balance: {self.get_balance()} ETH \n' \
f'Reason: {self.base_message} \n' \
f'Transaction: {self.payload}'
return message
def get_balance(self):
blockchain = BlockchainInterfaceFactory.get_interface()
balance = blockchain.client.get_balance(account=self.payload['from'])
return balance
@property
def insufficient_eth(self) -> str:
gas = (self.payload.get('gas', 1) * self.payload['gasPrice']) # FIXME: If gas is not included...
cost = gas + self.payload.get('value', 0)
message = f'{self.payload} from {self.payload["from"][:8]} - {self.base_message}.' \
f'Calculated cost is {cost} but sender only has {self.get_balance()}.'
return message
def __init__(self,
emitter = None, # TODO # 1754
poa: bool = None,
light: bool = False,
provider_process=NO_PROVIDER_PROCESS,
provider_uri: str = NO_BLOCKCHAIN_CONNECTION,
provider: Web3Providers = NO_BLOCKCHAIN_CONNECTION,
gas_strategy: Union[str, Callable] = DEFAULT_GAS_STRATEGY):
"""
TODO: #1502 - Move to API docs.
Filesystem Configuration Node Client EVM
================ ====================== =============== ===================== ===========================
Solidity Files -- SolidityCompiler - --- HTTPProvider ------ ...
| |
| |
| |
- *BlockchainInterface* -- IPCProvider ----- External EVM (geth, parity...)
| |
| |
TestProvider ----- EthereumTester -------------
|
|
PyEVM (Development Chain)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Runtime Files -- --BlockchainInterface ----> Registry
| | ^
| | |
| | |
Key Files ------ CharacterConfiguration Agent ... (Contract API)
| | ^
| | |
| | |
| | Actor ...Blockchain-Character API)
| | ^
| | |
| | |
Config File --- --------- Character ... (Public API)
^
|
Human
The Blockchain is the junction of the solidity compiler, a contract registry, and a collection of
web3 network providers as a means of interfacing with the ethereum blockchain to execute
or deploy contract code on the network.
Compiler and Registry Usage
-----------------------------
Contracts are freshly re-compiled if an instance of SolidityCompiler is passed; otherwise,
The registry will read contract data saved to disk that is be used to retrieve contact address and op-codes.
Optionally, A registry instance can be passed instead.
Provider Usage
---------------
https: // github.com / ethereum / eth - tester # available-backends
* HTTP Provider - Web3 HTTP provider, typically JSON RPC 2.0 over HTTP
* Websocket Provider - Web3 WS provider, typically JSON RPC 2.0 over WS, supply endpoint uri and websocket=True
* IPC Provider - Web3 File based IPC provider transported over standard I/O
* Custom Provider - A pre-initialized web3.py provider instance to attach to this interface
"""
self.log = Logger('Blockchain')
self.poa = poa
self.provider_uri = provider_uri
self._provider = provider
self._provider_process = provider_process
self.w3 = NO_BLOCKCHAIN_CONNECTION
self.client = NO_BLOCKCHAIN_CONNECTION # type: EthereumClient
self.transacting_power = READ_ONLY_INTERFACE
self.is_light = light
self.gas_strategy = self.get_gas_strategy(gas_strategy)
def __repr__(self):
r = '{name}({uri})'.format(name=self.__class__.__name__, uri=self.provider_uri)
return r
@classmethod
def from_dict(cls, payload: dict, **overrides) -> 'BlockchainInterface':
payload.update({k: v for k, v in overrides.items() if v is not None})
blockchain = cls(**payload)
return blockchain
def to_dict(self) -> dict:
payload = dict(provider_uri=self.provider_uri, poa=self.poa, light=self.is_light)
return payload
@property
def is_connected(self) -> bool:
"""
https://web3py.readthedocs.io/en/stable/__provider.html#examples-using-automated-detection
"""
if self.client is NO_BLOCKCHAIN_CONNECTION:
return False
return self.client.is_connected
@classmethod
def get_gas_strategy(cls, gas_strategy: Union[str, Callable] = None) -> Callable:
try:
gas_strategy = cls.GAS_STRATEGIES[gas_strategy]
except KeyError:
if gas_strategy:
if not callable(gas_strategy):
raise ValueError(f"{gas_strategy} must be callable to be a valid gas strategy.")
else:
gas_strategy = cls.GAS_STRATEGIES[cls.DEFAULT_GAS_STRATEGY]
return gas_strategy
def attach_middleware(self):
chain_id = int(self.client.chain_id)
if self.poa is None: # If POA is not set explicitly, try to autodetect from chain id
self.poa = chain_id in POA_CHAINS
self.log.debug(f'Ethereum chain: {self.client.chain_name} (chain_id={chain_id}, poa={self.poa})')
# For use with Proof-Of-Authority test-blockchains
if self.poa is True:
self.log.debug('Injecting POA middleware at layer 0')
self.client.inject_middleware(geth_poa_middleware, layer=0)
# Gas Price Strategy:
# Bundled web3 strategies are too expensive for Infura (it takes ~1 minute to get a price),
# so we use external gas price oracles, instead (see #2139)
if isinstance(self.client, InfuraClient):
gas_strategy = datafeed_fallback_gas_price_strategy
else:
gas_strategy = self.gas_strategy
self.client.set_gas_strategy(gas_strategy=gas_strategy)
gwei_gas_price = Web3.fromWei(self.client.gas_price_for_transaction(), 'gwei')
self.log.debug(f"Currently, our gas strategy returns a gas price of {gwei_gas_price} gwei")
self.client.add_middleware(middleware.time_based_cache_middleware)
self.client.add_middleware(middleware.latest_block_based_cache_middleware)
self.client.add_middleware(middleware.simple_cache_middleware)
def connect(self):
# Spawn child process
if self._provider_process:
self._provider_process.start()
provider_uri = self._provider_process.provider_uri(scheme='file')
else:
provider_uri = self.provider_uri
self.log.info(f"Using external Web3 Provider '{self.provider_uri}'")
# Attach Provider
self._attach_provider(provider=self._provider, provider_uri=provider_uri)
self.log.info("Connecting to {}".format(self.provider_uri))
if self._provider is NO_BLOCKCHAIN_CONNECTION:
raise self.NoProvider("There are no configured blockchain providers")
# Connect if not connected
try:
self.w3 = self.Web3(provider=self._provider)
self.client = EthereumClient.from_w3(w3=self.w3)
except requests.ConnectionError: # RPC
raise self.ConnectionFailed(f'Connection Failed - {str(self.provider_uri)} - is RPC enabled?')
except FileNotFoundError: # IPC File Protocol
raise self.ConnectionFailed(f'Connection Failed - {str(self.provider_uri)} - is IPC enabled?')
else:
self.attach_middleware()
return self.is_connected
def sync(self, emitter=None) -> None:
sync_state = self.client.sync()
if emitter is not None:
emitter.echo(f"Syncing: {self.client.chain_name.capitalize()}. Waiting for sync to begin.", verbosity=1)
while not len(self.client.peers):
emitter.echo("waiting for peers...", verbosity=1)
time.sleep(5)
peer_count = len(self.client.peers)
emitter.echo(
f"Found {'an' if peer_count == 1 else peer_count} Ethereum peer{('s' if peer_count > 1 else '')}.",
verbosity=1)
try:
emitter.echo("Beginning sync...", verbosity=1)
initial_state = next(sync_state)
except StopIteration: # will occur if no syncing needs to happen
emitter.echo("Local blockchain data is already synced.", verbosity=1)
return
prior_state = initial_state
total_blocks_to_sync = int(initial_state.get('highestBlock', 0)) - int(
initial_state.get('currentBlock', 0))
with click.progressbar(
length=total_blocks_to_sync,
label="sync progress",
file=emitter.get_stream(verbosity=1)
) as bar:
for syncdata in sync_state:
if syncdata:
blocks_accomplished = int(syncdata['currentBlock']) - int(
prior_state.get('currentBlock', 0))
bar.update(blocks_accomplished)
prior_state = syncdata
else:
try:
for syncdata in sync_state:
self.client.log.info(f"Syncing {syncdata['currentBlock']}/{syncdata['highestBlock']}")
except TypeError: # it's already synced
return
return
@property
def provider(self) -> Union[IPCProvider, WebsocketProvider, HTTPProvider]:
return self._provider
def _attach_provider(self,
provider: Web3Providers = None,
provider_uri: str = None) -> None:
"""
https://web3py.readthedocs.io/en/latest/providers.html#providers
"""
if not provider_uri and not provider:
raise self.NoProvider("No URI or provider instances supplied.")
if provider_uri and not provider:
uri_breakdown = urlparse(provider_uri)
if uri_breakdown.scheme == 'tester':
providers = {
'pyevm': _get_pyevm_test_provider,
'geth': _get_test_geth_parity_provider,
'parity-ethereum': _get_test_geth_parity_provider,
'mock': _get_mock_test_provider
}
provider_scheme = uri_breakdown.netloc
else:
providers = {
'auto': _get_auto_provider,
'infura': _get_infura_provider,
'ipc': _get_IPC_provider,
'file': _get_IPC_provider,
'ws': _get_websocket_provider,
'wss': _get_websocket_provider,
'http': _get_HTTP_provider,
'https': _get_HTTP_provider,
}
provider_scheme = uri_breakdown.scheme
# auto-detect for file based ipc
if not provider_scheme:
if os.path.exists(provider_uri):
# file is available - assume ipc/file scheme
provider_scheme = 'file'
self.log.info(f"Auto-detected provider scheme as 'file://' for provider {provider_uri}")
try:
self._provider = providers[provider_scheme](provider_uri)
except KeyError:
raise self.UnsupportedProvider(f"{provider_uri} is an invalid or unsupported blockchain provider URI")
else:
self.provider_uri = provider_uri or NO_BLOCKCHAIN_CONNECTION
else:
self._provider = provider
def __transaction_failed(self,
exception: Exception,
transaction_dict: dict,
contract_function: Union[ContractFunction, ContractConstructor]
) -> None:
"""
Re-raising error handler and context manager for transaction broadcast or
build failure events at the interface layer. This method is a last line of defense
against unhandled exceptions caused by transaction failures and must raise an exception.
# TODO: #1504 - Additional Handling of validation failures (gas limits, invalid fields, etc.)
"""
try:
# Assume this error is formatted as an IPC response
code, message = exception.args[0].values()
except (ValueError, IndexError, AttributeError):
# TODO: #1504 - Try even harder to determine if this is insufficient funds causing the issue,
# This may be best handled at the agent or actor layer for registry and token interactions.
# Worst case scenario - raise the exception held in context implicitly
raise exception
else:
if int(code) != self.TransactionFailed.IPC_CODE:
# Only handle client-specific exceptions
# https://www.jsonrpc.org/specification Section 5.1
raise exception
self.log.critical(message) # simple context
transaction_failed = self.TransactionFailed(message=message, # rich error (best case)
contract_function=contract_function,
transaction_dict=transaction_dict)
raise transaction_failed from exception
def __log_transaction(self, transaction_dict: dict, contract_function: ContractFunction):
"""
Format and log a transaction dict and return the transaction name string.
This method *must not* mutate the original transaction dict.
"""
# Do not mutate the original transaction dict
tx = dict(transaction_dict).copy()
# Format
if tx.get('to'):
tx['to'] = to_checksum_address(contract_function.address)
try:
tx['selector'] = contract_function.selector
except AttributeError:
pass
tx['from'] = to_checksum_address(tx['from'])
tx.update({f: prettify_eth_amount(v) for f, v in tx.items() if f in ('gasPrice', 'value')})
payload_pprint = ', '.join("{}: {}".format(k, v) for k, v in tx.items())
# Log
transaction_name = get_transaction_name(contract_function=contract_function)
self.log.debug(f"[TX-{transaction_name}] | {payload_pprint}")
@validate_checksum_address
def build_payload(self,
sender_address: str,
payload: dict = None,
transaction_gas_limit: int = None,
) -> dict:
base_payload = {'chainId': int(self.client.chain_id),
'nonce': self.client.w3.eth.getTransactionCount(sender_address, 'pending'),
'from': sender_address}
# Aggregate
if not payload:
payload = {}
payload.update(base_payload)
# Explicit gas override - will skip gas estimation in next operation.
if transaction_gas_limit:
payload['gas'] = int(transaction_gas_limit)
return payload
@validate_checksum_address
def build_contract_transaction(self,
contract_function: ContractFunction,
sender_address: str,
payload: dict = None,
transaction_gas_limit: int = None,
) -> dict:
payload = self.build_payload(sender_address=sender_address,
payload=payload,
transaction_gas_limit=transaction_gas_limit)
self.__log_transaction(transaction_dict=payload, contract_function=contract_function)
try:
transaction_dict = contract_function.buildTransaction(payload) # Gas estimation occurs here
except (TestTransactionFailed, ValidationError, ValueError) as error:
# Note: Geth raises ValueError in the same condition that pyevm raises ValidationError here.
# Treat this condition as "Transaction Failed" during gas estimation.
raise self.__transaction_failed(exception=error, transaction_dict=payload, contract_function=contract_function)
return transaction_dict
def sign_and_broadcast_transaction(self,
transaction_dict,
transaction_name: str = "",
confirmations: int = 0
) -> dict:
#
# Setup
#
# TODO # 1754 - Move this to singleton - I do not approve... nor does Bogdan?
if GlobalLoggerSettings._json_ipc:
emitter = JSONRPCStdoutEmitter()
else:
emitter = StdoutEmitter()
if self.transacting_power is READ_ONLY_INTERFACE:
raise self.InterfaceError(str(READ_ONLY_INTERFACE))
#
# Sign
#
# TODO: Show the USD Price: https://api.coinmarketcap.com/v1/ticker/ethereum/
price = transaction_dict['gasPrice']
price_gwei = Web3.fromWei(price, 'gwei')
cost_wei = price * transaction_dict['gas']
cost = Web3.fromWei(cost_wei, 'ether')
if self.transacting_power.is_device:
emitter.message(f'Confirm transaction {transaction_name} on hardware wallet... '
f'({cost} ETH @ {price_gwei} gwei)',
color='yellow')
signed_raw_transaction = self.transacting_power.sign_transaction(transaction_dict)
#
# Broadcast
#
emitter.message(f'Broadcasting {transaction_name} Transaction ({cost} ETH @ {price_gwei} gwei)...',
color='yellow')
try:
txhash = self.client.send_raw_transaction(signed_raw_transaction) # <--- BROADCAST
except (TestTransactionFailed, ValueError) as error:
raise # TODO: Unify with Transaction failed handling
#
# Receipt
#
try: # TODO: Handle block confirmation exceptions
receipt = self.client.wait_for_receipt(txhash, timeout=self.TIMEOUT, confirmations=confirmations)
except TimeExhausted:
# TODO: #1504 - Handle transaction timeout
raise
else:
self.log.debug(f"[RECEIPT-{transaction_name}] | txhash: {receipt['transactionHash'].hex()}")
#
# Confirmations
#
# Primary check
transaction_status = receipt.get('status', UNKNOWN_TX_STATUS)
if transaction_status == 0:
failure = f"Transaction transmitted, but receipt returned status code 0. " \
f"Full receipt: \n {pprint.pformat(receipt, indent=2)}"
raise self.InterfaceError(failure)
if transaction_status is UNKNOWN_TX_STATUS:
self.log.info(f"Unknown transaction status for {txhash} (receipt did not contain a status field)")
# Secondary check
tx = self.client.get_transaction(txhash)
if tx["gas"] == receipt["gasUsed"]:
raise self.InterfaceError(f"Transaction consumed 100% of transaction gas."
f"Full receipt: \n {pprint.pformat(receipt, indent=2)}")
return receipt
def get_blocktime(self):
return self.client.get_blocktime()
@validate_checksum_address
def send_transaction(self,
contract_function: Union[ContractFunction, ContractConstructor],
sender_address: str,
payload: dict = None,
transaction_gas_limit: int = None,
confirmations: int = 0
) -> dict:
transaction = self.build_contract_transaction(contract_function=contract_function,
sender_address=sender_address,
payload=payload,
transaction_gas_limit=transaction_gas_limit)
# Get transaction name
try:
transaction_name = contract_function.fn_name.upper()
except AttributeError:
transaction_name = 'DEPLOY' if isinstance(contract_function, ContractConstructor) else 'UNKNOWN'
receipt = self.sign_and_broadcast_transaction(transaction_dict=transaction,
transaction_name=transaction_name,
confirmations=confirmations)
return receipt
def get_contract_by_name(self,
registry: BaseContractRegistry,
contract_name: str,
contract_version: str = None,
enrollment_version: Union[int, str] = None,
proxy_name: str = None,
use_proxy_address: bool = True
) -> VersionedContract:
"""
Instantiate a deployed contract from registry data,
and assimilate it with its proxy if it is upgradeable.
"""
target_contract_records = registry.search(contract_name=contract_name, contract_version=contract_version)
if not target_contract_records:
raise self.UnknownContract(f"No such contract records with name {contract_name}:{contract_version}.")
if proxy_name:
# Lookup proxies; Search for a published proxy that targets this contract record
proxy_records = registry.search(contract_name=proxy_name)
results = list()
for proxy_name, proxy_version, proxy_address, proxy_abi in proxy_records:
proxy_contract = self.client.w3.eth.contract(abi=proxy_abi,
address=proxy_address,
version=proxy_version,
ContractFactoryClass=self._contract_factory)
# Read this dispatcher's target address from the blockchain
proxy_live_target_address = proxy_contract.functions.target().call()
for target_name, target_version, target_address, target_abi in target_contract_records:
if target_address == proxy_live_target_address:
if use_proxy_address:
triplet = (proxy_address, target_version, target_abi)
else:
triplet = (target_address, target_version, target_abi)
else:
continue
results.append(triplet)
if len(results) > 1:
address, _version, _abi = results[0]
message = "Multiple {} deployments are targeting {}".format(proxy_name, address)
raise self.InterfaceError(message.format(contract_name))
else:
try:
selected_address, selected_version, selected_abi = results[0]
except IndexError:
raise self.UnknownContract(
f"There are no Dispatcher records targeting '{contract_name}':{contract_version}")
else:
# TODO: use_proxy_address doesnt' work in this case. Should we raise if used?
# NOTE: 0 must be allowed as a valid version number
if len(target_contract_records) != 1:
if enrollment_version is None:
m = f"{len(target_contract_records)} records enrolled " \
f"for contract {contract_name}:{contract_version} " \
f"and no version index was supplied."
raise self.InterfaceError(m)
enrollment_version = self.__get_enrollment_version_index(name=contract_name,
contract_version=contract_version,
version_index=enrollment_version,
enrollments=len(target_contract_records))
else:
enrollment_version = -1 # default
_contract_name, selected_version, selected_address, selected_abi = target_contract_records[enrollment_version]
# Create the contract from selected sources
unified_contract = self.client.w3.eth.contract(abi=selected_abi,
address=selected_address,
version=selected_version,
ContractFactoryClass=self._contract_factory)
return unified_contract
@staticmethod
def __get_enrollment_version_index(version_index: Union[int, str],
enrollments: int,
name: str,
contract_version: str):
version_names = {'latest': -1, 'earliest': 0}
try:
version = version_names[version_index]
except KeyError:
try:
version = int(version_index)
except ValueError:
what_is_this = version_index
raise ValueError(f"'{what_is_this}' is not a valid enrollment version number")
else:
if version > enrollments - 1:
message = f"Version index '{version}' is larger than the number of enrollments " \
f"for {name}:{contract_version}."
raise ValueError(message)
return version
class WorkTracker:
CLOCK = reactor
INTERVAL_FLOOR = 60 * 15 # fifteen minutes
INTERVAL_CEIL = 60 * 180 # three hours
ALLOWED_DEVIATION = 0.5 # i.e., up to +50% from the expected confirmation time
def __init__(self, worker, *args, **kwargs):
super().__init__(*args, **kwargs)
self.log = Logger('stake-tracker')
self.worker = worker
self.staking_agent = self.worker.staking_agent
self.client = self.staking_agent.blockchain.client
self.gas_strategy = self.staking_agent.blockchain.gas_strategy
self._tracking_task = task.LoopingCall(self._do_work)
self._tracking_task.clock = self.CLOCK
self.__pending = dict() # TODO: Prime with pending worker transactions
self.__requirement = None
self.__current_period = None
self.__start_time = NOT_STAKING
self.__uptime_period = NOT_STAKING
self._abort_on_error = False
@classmethod
def random_interval(cls) -> int:
return random.randint(cls.INTERVAL_FLOOR, cls.INTERVAL_CEIL)
@property
def current_period(self):
return self.__current_period
def max_confirmation_time(self) -> int:
expected_time = EXPECTED_CONFIRMATION_TIME_IN_SECONDS[self.gas_strategy]
result = expected_time * (1 + self.ALLOWED_DEVIATION)
return result
def stop(self) -> None:
if self._tracking_task.running:
self._tracking_task.stop()
self.log.info(f"STOPPED WORK TRACKING")
def start(self, act_now: bool = True, requirement_func: Callable = None, force: bool = False) -> None:
"""
High-level stake tracking initialization, this function aims
to be safely called at any time - For example, it is okay to call
this function multiple times within the same period.
"""
if self._tracking_task.running and not force:
return
# Add optional confirmation requirement callable
self.__requirement = requirement_func
# Record the start time and period
self.__start_time = maya.now()
self.__uptime_period = self.staking_agent.get_current_period()
self.__current_period = self.__uptime_period
self.log.info(f"START WORK TRACKING")
d = self._tracking_task.start(interval=self.random_interval(), now=act_now)
d.addErrback(self.handle_working_errors)
def _crash_gracefully(self, failure=None) -> None:
"""
A facility for crashing more gracefully in the event that
an exception is unhandled in a different thread.
"""
self._crashed = failure
failure.raiseException()
def handle_working_errors(self, *args, **kwargs) -> None:
failure = args[0]
if self._abort_on_error:
self.log.critical(f'Unhandled error during node work tracking. {failure!r}',
failure=failure)
self.stop()
reactor.callFromThread(self._crash_gracefully, failure=failure)
else:
self.log.warn(f'Unhandled error during work tracking: {failure.getTraceback()!r}',
failure=failure)
self.start()
def __work_requirement_is_satisfied(self) -> bool:
# TODO: Check for stake expiration and exit
if self.__requirement is None:
return True
r = self.__requirement()
if not isinstance(r, bool):
raise ValueError(f"'requirement' must return a boolean.")
return r
@property
def pending(self) -> Dict[int, HexBytes]:
return self.__pending.copy()
def __commitments_tracker_is_consistent(self) -> bool:
worker_address = self.worker.worker_address
tx_count_pending = self.client.get_transaction_count(account=worker_address, pending=True)
tx_count_latest = self.client.get_transaction_count(account=worker_address, pending=False)
txs_in_mempool = tx_count_pending - tx_count_latest
if len(self.__pending) == txs_in_mempool:
return True # OK!
if txs_in_mempool > len(self.__pending): # We're missing some pending TXs
return False
else: # TODO #2429: What to do when txs_in_mempool < len(self.__pending)? What does this imply?
return True
def __track_pending_commitments(self) -> bool:
# TODO: Keep a purpose-built persistent log of worker transaction history
unmined_transactions = list()
pending_transactions = self.pending.items() # note: this must be performed non-mutatively
for tx_firing_block_number, txhash in sorted(pending_transactions):
try:
confirmed_tx_receipt = self.client.get_transaction_receipt(transaction_hash=txhash)
except TransactionNotFound:
unmined_transactions.append(txhash) # mark as unmined - Keep tracking it for now
continue
else:
confirmation_block_number = confirmed_tx_receipt['blockNumber']
confirmations = confirmation_block_number - tx_firing_block_number
self.log.info(f'Commitment transaction {txhash.hex()[:10]} confirmed: {confirmations} confirmations')
del self.__pending[tx_firing_block_number]
if unmined_transactions:
s = "s" if len(unmined_transactions) > 1 else ""
self.log.info(f'{len(unmined_transactions)} pending commitment transaction{s} detected.')
inconsistent_tracker = not self.__commitments_tracker_is_consistent()
if inconsistent_tracker:
# If we detect there's a mismatch between the number of internally tracked and
# pending block transactions, create a special pending TX that accounts for this.
# TODO: Detect if this untracked pending transaction is a commitment transaction at all.
self.__pending[0] = UNTRACKED_PENDING_TRANSACTION
return True
return bool(self.__pending)
def __fire_replacement_commitment(self, current_block_number: int, tx_firing_block_number: int) -> None:
replacement_txhash = self.__fire_commitment() # replace
self.__pending[current_block_number] = replacement_txhash # track this transaction
del self.__pending[tx_firing_block_number] # assume our original TX is stuck
def __handle_replacement_commitment(self, current_block_number: int) -> None:
tx_firing_block_number, txhash = list(sorted(self.pending.items()))[0]
if txhash is UNTRACKED_PENDING_TRANSACTION:
# TODO: Detect if this untracked pending transaction is a commitment transaction at all.
message = f"We have an untracked pending transaction. Issuing a replacement transaction."
else:
# If the transaction is still not mined after a max confirmation time
# (based on current gas strategy) issue a replacement transaction.
wait_time_in_blocks = current_block_number - tx_firing_block_number
wait_time_in_seconds = wait_time_in_blocks * AVERAGE_BLOCK_TIME_IN_SECONDS
if wait_time_in_seconds < self.max_confirmation_time():
self.log.info(f'Waiting for pending commitment transaction to be mined ({txhash.hex()}).')
return
else:
message = f"We've waited for {wait_time_in_seconds}, but max time is {self.max_confirmation_time()}" \
f" for {self.gas_strategy} gas strategy. Issuing a replacement transaction."
# Send a replacement transaction
self.log.info(message)
self.__fire_replacement_commitment(current_block_number=current_block_number,
tx_firing_block_number=tx_firing_block_number)
def _do_work(self) -> None:
"""
Async working task for Ursula # TODO: Split into multiple async tasks
"""
# Call once here, and inject later for temporal consistency
current_block_number = self.client.block_number
# Update on-chain status
self.log.info(f"Checking for new period. Current period is {self.__current_period}")
onchain_period = self.staking_agent.get_current_period() # < -- Read from contract
if self.current_period != onchain_period:
self.__current_period = onchain_period
# TODO: #1515 and #1517 - Shut down at end of terminal stake
# This slows down tests substantially and adds additional
# RPC calls, but might be acceptable in production
# self.worker.stakes.refresh()
# Measure working interval
interval = onchain_period - self.worker.last_committed_period
if interval < 0:
return # No need to commit to this period. Save the gas.
if interval > 0:
# TODO: #1516 Follow-up actions for missed commitments
self.log.warn(f"MISSED COMMITMENTS - {interval} missed staking commitments detected.")
# Commitment tracking
unmined_transactions = self.__track_pending_commitments()
if unmined_transactions:
self.__handle_replacement_commitment(current_block_number=current_block_number)
# while there are known pending transactions, remain in fast interval mode
self._tracking_task.interval = self.INTERVAL_FLOOR
return # This cycle is finished.
else:
# Randomize the next task interval over time, within bounds.
self._tracking_task.interval = self.random_interval()
# Only perform work this round if the requirements are met
if not self.__work_requirement_is_satisfied():
self.log.warn(f'COMMIT PREVENTED (callable: "{self.__requirement.__name__}") - '
f'There are unmet commit requirements.')
# TODO: Follow-up actions for failed requirements
return
txhash = self.__fire_commitment()
self.__pending[current_block_number] = txhash
def __fire_commitment(self):
"""Makes an initial/replacement worker commitment transaction"""
transacting_power = self.worker.transacting_power
with transacting_power:
txhash = self.worker.commit_to_next_period(fire_and_forget=True) # < --- blockchain WRITE
self.log.info(f"Making a commitment to period {self.current_period} - TxHash: {txhash.hex()}")
return txhash
mock_remember_node, mock_rest_app_creation, mock_secret_source,
mock_verify_node)
from tests.utils.blockchain import TesterBlockchain, token_airdrop
from tests.utils.config import (make_alice_test_configuration,
make_bob_test_configuration,
make_ursula_test_configuration)
from tests.utils.middleware import MockRestMiddleware, MockRestMiddlewareForLargeFleetTests
from tests.utils.policy import generate_random_label
from tests.utils.ursula import (MOCK_URSULA_STARTING_PORT,
make_decentralized_ursulas,
make_federated_ursulas,
MOCK_KNOWN_URSULAS_CACHE,
_mock_ursula_reencrypts)
from constant_sorrow.constants import (FULL, INIT)
test_logger = Logger("test-logger")
# defer.setDebugging(True)
#
# Temporary
#
@pytest.fixture(scope="function")
def tempfile_path():
fd, path = tempfile.mkstemp()
yield path
os.close(fd)
os.remove(path)
class StakeList(UserList):
@validate_checksum_address
def __init__(self,
registry: BaseContractRegistry,
checksum_address: str = None,
*args, **kwargs):
super().__init__(*args, **kwargs)
self.log = Logger('stake-tracker')
self.staking_agent = ContractAgency.get_agent(StakingEscrowAgent, registry=registry)
from nucypher.blockchain.economics import EconomicsFactory
self.economics = EconomicsFactory.get_economics(registry=registry)
self.__initial_period = NOT_STAKING
self.__terminal_period = NOT_STAKING
# "load-in": Read on-chain stakes
# Allow stake tracker to be initialized as an empty collection.
if checksum_address:
if not is_checksum_address(checksum_address):
raise ValueError(f'{checksum_address} is not a valid EIP-55 checksum address')
self.checksum_address = checksum_address
self.__updated = None
@property
def updated(self) -> maya.MayaDT:
return self.__updated
@property
def initial_period(self) -> int:
return self.__initial_period
@property
def terminal_period(self) -> int:
return self.__terminal_period
@validate_checksum_address
def refresh(self) -> None:
"""Public staking cache invalidation method"""
return self.__read_stakes()
def __read_stakes(self) -> None:
"""Rewrite the local staking cache by reading on-chain stakes"""
existing_records = len(self)
# Candidate replacement cache values
current_period = self.staking_agent.get_current_period()
onchain_stakes, initial_period, terminal_period = list(), 0, current_period
# Read from blockchain
stakes_reader = self.staking_agent.get_all_stakes(staker_address=self.checksum_address)
for onchain_index, stake_info in enumerate(stakes_reader):
if not stake_info:
onchain_stake = EMPTY_STAKING_SLOT
else:
onchain_stake = Stake.from_stake_info(checksum_address=self.checksum_address,
stake_info=stake_info,
staking_agent=self.staking_agent,
index=onchain_index,
economics=self.economics)
# rack the earliest terminal period
if onchain_stake.first_locked_period:
if onchain_stake.first_locked_period < initial_period:
initial_period = onchain_stake.first_locked_period
# rack the latest terminal period
if onchain_stake.final_locked_period > terminal_period:
terminal_period = onchain_stake.final_locked_period
# Store the replacement stake
onchain_stakes.append(onchain_stake)
# Commit the new stake and terminal values to the cache
self.data = onchain_stakes
if onchain_stakes:
self.__initial_period = initial_period
self.__terminal_period = terminal_period
changed_records = abs(existing_records - len(onchain_stakes))
self.log.debug(f"Updated {changed_records} local staking cache entries.")
# Record most recent cache update
self.__updated = maya.now()
def __init__(
self,
emitter=None, # TODO # 1754
poa: bool = None,
light: bool = False,
provider_uri: str = NO_BLOCKCHAIN_CONNECTION,
provider: BaseProvider = NO_BLOCKCHAIN_CONNECTION,
gas_strategy: Optional[Union[str, Callable]] = None,
max_gas_price: Optional[int] = None):
"""
TODO: #1502 - Move to API docs.
Filesystem Configuration Node Client EVM
================ ====================== =============== ===================== ===========================
Solidity Files -- SolidityCompiler - --- HTTPProvider ------ ...
| |
| |
| |
- *BlockchainInterface* -- IPCProvider ----- External EVM (geth, parity...)
| |
| |
TestProvider ----- EthereumTester -------------
|
|
PyEVM (Development Chain)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Runtime Files -- --BlockchainInterface ----> Registry
| | ^
| | |
| | |
Key Files ------ CharacterConfiguration Agent ... (Contract API)
| | ^
| | |
| | |
| | Actor ...Blockchain-Character API)
| | ^
| | |
| | |
Config File --- --------- Character ... (Public API)
^
|
Human
The Blockchain is the junction of the solidity compiler, a contract registry, and a collection of
web3 network providers as a means of interfacing with the ethereum blockchain to execute
or deploy contract code on the network.
Compiler and Registry Usage
-----------------------------
Contracts are freshly re-compiled if an instance of SolidityCompiler is passed; otherwise,
The registry will read contract data saved to disk that is be used to retrieve contact address and op-codes.
Optionally, A registry instance can be passed instead.
Provider Usage
---------------
https: // github.com / ethereum / eth - tester # available-backends
* HTTP Provider - Web3 HTTP provider, typically JSON RPC 2.0 over HTTP
* Websocket Provider - Web3 WS provider, typically JSON RPC 2.0 over WS, supply endpoint uri and websocket=True
* IPC Provider - Web3 File based IPC provider transported over standard I/O
* Custom Provider - A pre-initialized web3.py provider instance to attach to this interface
"""
self.log = Logger('Blockchain')
self.poa = poa
self.provider_uri = provider_uri
self._provider = provider
self.w3 = NO_BLOCKCHAIN_CONNECTION
self.client = NO_BLOCKCHAIN_CONNECTION
self.is_light = light
# TODO: Not ready to give users total flexibility. Let's stick for the moment to known values. See #2447
if gas_strategy not in ('slow', 'medium', 'fast', 'free',
None): # FIXME: What is 'None' doing here?
raise ValueError(f"'{gas_strategy}' is an invalid gas strategy")
self.gas_strategy = gas_strategy or self.DEFAULT_GAS_STRATEGY
self.max_gas_price = max_gas_price
class RestMiddleware:
log = Logger()
_client_class = NucypherMiddlewareClient
class UnexpectedResponse(Exception):
def __init__(self, message, status, *args, **kwargs):
super().__init__(message, *args, **kwargs)
self.status = status
class NotFound(UnexpectedResponse):
def __init__(self, *args, **kwargs):
super().__init__(status=404, *args, **kwargs)
class BadRequest(UnexpectedResponse):
def __init__(self, reason, *args, **kwargs):
self.reason = reason
super().__init__(message=reason, status=400, *args, **kwargs)
def __init__(self, registry=None):
self.client = self._client_class(registry)
def get_certificate(self,
host,
port,
timeout=3,
retry_attempts: int = 3,
retry_rate: int = 2,
current_attempt: int = 0):
socket.setdefaulttimeout(timeout) # Set Socket Timeout
try:
self.log.info(f"Fetching seednode {host}:{port} TLS certificate")
seednode_certificate = ssl.get_server_certificate(addr=(host,
port))
except socket.timeout:
if current_attempt == retry_attempts:
message = f"No Response from seednode {host}:{port} after {retry_attempts} attempts"
self.log.info(message)
raise ConnectionRefusedError("No response from {}:{}".format(
host, port))
self.log.info(
f"No Response from seednode {host}:{port}. Retrying in {retry_rate} seconds..."
)
time.sleep(retry_rate)
return self.get_certificate(host, port, timeout, retry_attempts,
retry_rate, current_attempt + 1)
except OSError:
raise # TODO: #1835
else:
certificate = x509.load_pem_x509_certificate(
seednode_certificate.encode(), backend=default_backend())
return certificate
def consider_arrangement(self, arrangement):
node = arrangement.ursula
response = self.client.post(node_or_sprout=node,
path="consider_arrangement",
data=bytes(arrangement),
timeout=2)
return response
def enact_policy(self, ursula, kfrag_id, payload):
response = self.client.post(node_or_sprout=ursula,
path=f'kFrag/{kfrag_id.hex()}',
data=payload,
timeout=2)
return response
def reencrypt(self, work_order):
ursula_rest_response = self.send_work_order_payload_to_ursula(
work_order)
splitter = BytestringSplitter((CapsuleFrag, VariableLengthBytestring),
Signature)
cfrags_and_signatures = splitter.repeat(ursula_rest_response.content)
return cfrags_and_signatures
def revoke_arrangement(self, ursula, revocation):
# TODO: Implement revocation confirmations
response = self.client.delete(
node_or_sprout=ursula,
path=f"kFrag/{revocation.arrangement_id.hex()}",
data=bytes(revocation),
)
return response
def get_competitive_rate(self):
return NotImplemented
def get_treasure_map_from_node(self, node, map_id):
response = self.client.get(node_or_sprout=node,
path=f"treasure_map/{map_id}",
timeout=2)
return response
def put_treasure_map_on_node(self, node, map_id, map_payload):
response = self.client.post(node_or_sprout=node,
path=f"treasure_map/{map_id}",
data=map_payload,
timeout=2)
return response
def send_work_order_payload_to_ursula(self, work_order):
payload = work_order.payload()
id_as_hex = work_order.arrangement_id.hex()
response = self.client.post(node_or_sprout=work_order.ursula,
path=f"kFrag/{id_as_hex}/reencrypt",
data=payload,
timeout=2)
return response
def check_rest_availability(self, initiator, responder):
response = self.client.post(
node_or_sprout=responder,
data=bytes(initiator),
path="ping",
timeout=6, # Two round trips are expected
)
return response
def get_nodes_via_rest(self,
node,
announce_nodes=None,
nodes_i_need=None,
fleet_checksum=None):
if nodes_i_need:
# TODO: This needs to actually do something. NRN
# Include node_ids in the request; if the teacher node doesn't know about the
# nodes matching these ids, then it will ask other nodes.
pass
if fleet_checksum:
params = {'fleet': fleet_checksum}
else:
params = {}
if announce_nodes:
payload = bytes().join(
bytes(VariableLengthBytestring(n)) for n in announce_nodes)
response = self.client.post(
node_or_sprout=node,
path="node_metadata",
params=params,
data=payload,
)
else:
response = self.client.get(node_or_sprout=node,
path="node_metadata",
params=params)
return response
def test_collect_rewards_integration(
click_runner, testerchain, agency_local_registry,
stakeholder_configuration_file_location, blockchain_alice,
blockchain_bob, random_policy_label, manual_staker, manual_worker,
token_economics, mock_transacting_power_activation, policy_value,
policy_rate):
half_stake_time = token_economics.minimum_locked_periods // 2 # Test setup
logger = Logger("Test-CLI") # Enter the Teacher's Logger, and
current_period = 0 # State the initial period for incrementing
staker_address = manual_staker
worker_address = manual_worker
staker = Staker(is_me=True,
checksum_address=staker_address,
registry=agency_local_registry)
staker.refresh_stakes()
# The staker is staking.
assert staker.is_staking
assert staker.stakes
assert staker.worker_address == worker_address
ursula_port = select_test_port()
ursula = Ursula(is_me=True,
checksum_address=staker_address,
worker_address=worker_address,
registry=agency_local_registry,
rest_host='127.0.0.1',
rest_port=ursula_port,
network_middleware=MockRestMiddleware(),
db_filepath=tempfile.mkdtemp())
MOCK_KNOWN_URSULAS_CACHE[ursula_port] = ursula
assert ursula.worker_address == worker_address
assert ursula.checksum_address == staker_address
mock_transacting_power_activation(account=worker_address,
password=INSECURE_DEVELOPMENT_PASSWORD)
# Make a commitment for half the first stake duration
for _ in range(half_stake_time):
logger.debug(
f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<")
ursula.commit_to_next_period()
testerchain.time_travel(periods=1)
current_period += 1
# Alice creates a policy and grants Bob access
blockchain_alice.selection_buffer = 1
M, N = 1, 1
days = 3
now = testerchain.w3.eth.getBlock(block_identifier='latest').timestamp
expiration = maya.MayaDT(now).add(days=days - 1)
blockchain_policy = blockchain_alice.grant(bob=blockchain_bob,
label=random_policy_label,
m=M,
n=N,
value=policy_value,
expiration=expiration,
handpicked_ursulas={ursula})
# Ensure that the handpicked Ursula was selected for the policy
arrangement = list(blockchain_policy._accepted_arrangements)[0]
assert arrangement.ursula == ursula
# Bob learns about the new staker and joins the policy
blockchain_bob.start_learning_loop()
blockchain_bob.remember_node(node=ursula)
blockchain_bob.join_policy(random_policy_label,
bytes(blockchain_alice.stamp))
# Enrico Encrypts (of course)
enrico = Enrico(policy_encrypting_key=blockchain_policy.public_key,
network_middleware=MockRestMiddleware())
verifying_key = blockchain_alice.stamp.as_umbral_pubkey()
for index in range(half_stake_time - 5):
logger.debug(
f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<")
ursula.commit_to_next_period()
# Encrypt
random_data = os.urandom(random.randrange(20, 100))
ciphertext, signature = enrico.encrypt_message(plaintext=random_data)
# Decrypt
cleartexts = blockchain_bob.retrieve(ciphertext,
enrico=enrico,
alice_verifying_key=verifying_key,
label=random_policy_label)
assert random_data == cleartexts[0]
# Ursula Staying online and the clock advancing
testerchain.time_travel(periods=1)
current_period += 1
# Finish the passage of time for the first Stake
for _ in range(5): # plus the extended periods from stake division
logger.debug(
f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<")
ursula.commit_to_next_period()
testerchain.time_travel(periods=1)
current_period += 1
#
# WHERES THE MONEY URSULA?? - Collecting Rewards
#
# The address the client wants Ursula to send rewards to
burner_wallet = testerchain.w3.eth.account.create(
INSECURE_DEVELOPMENT_PASSWORD)
# The rewards wallet is initially empty, because it is freshly created
assert testerchain.client.get_balance(burner_wallet.address) == 0
# Rewards will be unlocked after the
# final committed period has passed (+1).
logger.debug(f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<")
testerchain.time_travel(periods=1)
current_period += 1
logger.debug(f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<")
# At least half of the tokens are unlocked (restaking was enabled for some prior periods)
assert staker.locked_tokens() >= token_economics.minimum_allowed_locked
# Since we are mocking the blockchain connection, manually consume the transacting power of the Staker.
mock_transacting_power_activation(account=staker_address,
password=INSECURE_DEVELOPMENT_PASSWORD)
# Collect Policy Fee
collection_args = ('stake', 'collect-reward', '--config-file',
stakeholder_configuration_file_location, '--policy-fee',
'--no-staking-reward', '--staking-address',
staker_address, '--withdraw-address',
burner_wallet.address)
result = click_runner.invoke(nucypher_cli,
collection_args,
input=INSECURE_DEVELOPMENT_PASSWORD,
catch_exceptions=False)
assert result.exit_code == 0
# Policy Fee
collected_policy_fee = testerchain.client.get_balance(
burner_wallet.address)
expected_collection = policy_rate * 30
assert collected_policy_fee == expected_collection
# Finish the passage of time... once and for all
# Extended periods from stake division
for _ in range(9):
ursula.commit_to_next_period()
current_period += 1
logger.debug(
f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<")
testerchain.time_travel(periods=1)
#
# Collect Staking Reward
#
balance_before_collecting = staker.token_agent.get_balance(
address=staker_address)
collection_args = ('stake', 'collect-reward', '--config-file',
stakeholder_configuration_file_location,
'--no-policy-fee', '--staking-reward',
'--staking-address', staker_address, '--force')
result = click_runner.invoke(nucypher_cli,
collection_args,
input=INSECURE_DEVELOPMENT_PASSWORD,
catch_exceptions=False)
assert result.exit_code == 0
# The staker has withdrawn her staking rewards
assert staker.token_agent.get_balance(
address=staker_address) > balance_before_collecting
RequestErrors = (
# https://requests.readthedocs.io/en/latest/user/quickstart/#errors-and-exceptions
ConnectionError,
TimeoutError,
RequestException,
HTTPError
)
RESERVED_IP_ADDRESSES = (
'0.0.0.0',
'127.0.0.1',
'1.2.3.4'
)
IP_DETECTION_LOGGER = Logger('external-ip-detection')
def validate_worker_ip(worker_ip: str) -> None:
if worker_ip in RESERVED_IP_ADDRESSES:
raise InvalidWorkerIP(f'{worker_ip} is not a valid or permitted worker IP address. '
f'Verify the rest_host is set to the external IPV4 address')
def _request(url: str, certificate=None) -> Union[str, None]:
"""
Utility function to send a GET request to a URL returning it's
text content or None, suppressing all errors. Certificate is
needed if the remote URL source is self-signed.
"""