class EthereumClient: is_local = False GETH = 'Geth' PARITY = 'Parity' ALT_PARITY = 'Parity-Ethereum' GANACHE = 'EthereumJS TestRPC' ETHEREUM_TESTER = 'EthereumTester' # (PyEVM) CLEF = 'Clef' # Signer-only BLOCK_CONFIRMATIONS_POLLING_TIME = 3 # seconds TRANSACTION_POLLING_TIME = 0.5 # seconds COOLING_TIME = 5 # seconds STALECHECK_ALLOWABLE_DELAY = 30 # seconds class ConnectionNotEstablished(RuntimeError): pass class SyncTimeout(RuntimeError): pass class UnknownAccount(ValueError): pass class TransactionBroadcastError(RuntimeError): pass class NotEnoughConfirmations(TransactionBroadcastError): pass class TransactionTimeout(TransactionBroadcastError): pass class ChainReorganizationDetected(TransactionBroadcastError): """Raised when block confirmations logic detects that a TX was lost due to a chain reorganization""" error_message = ( "Chain re-organization detected: Transaction {transaction_hash} was reported to be in " "block {block_hash}, but it's not there anymore") def __init__(self, receipt): self.receipt = receipt self.message = self.error_message.format( transaction_hash=Web3.toHex(receipt['transactionHash']), block_hash=Web3.toHex(receipt['blockHash'])) super().__init__(self.message) def __init__(self, w3, node_technology: str, version: str, platform: str, backend: str): self.w3 = w3 self.node_technology = node_technology self.node_version = version self.platform = platform self.backend = backend self.log = Logger(self.__class__.__name__) self._add_default_middleware() def _add_default_middleware(self): # default retry functionality self.log.debug('Adding RPC retry middleware to client') self.add_middleware(RetryRequestMiddleware) @classmethod def _get_variant(cls, w3): return cls @classmethod def from_w3(cls, w3: Web3) -> 'EthereumClient': """ Client version strings: Geth -> 'Geth/v1.4.11-stable-fed692f6/darwin/go1.7' Parity -> 'Parity-Ethereum/v2.5.1-beta-e0141f8-20190510/x86_64-linux-gnu/rustc1.34.1' Ganache -> 'EthereumJS TestRPC/v2.1.5/ethereum-js' PyEVM -> 'EthereumTester/0.1.0b39/linux/python3.6.7' """ clients = { # Geth cls.GETH: GethClient, # Parity cls.PARITY: ParityClient, cls.ALT_PARITY: ParityClient, # Test Clients cls.GANACHE: GanacheClient, cls.ETHEREUM_TESTER: EthereumTesterClient, } try: client_data = w3.clientVersion.split('/') node_technology = client_data[0] ClientSubclass = clients[node_technology] except (ValueError, IndexError): raise ValueError( f"Invalid client version string. Got '{w3.clientVersion}'") except KeyError: raise NotImplementedError( f'{w3.clientVersion} is not a supported ethereum client') client_kwargs = { 'node_technology': node_technology, 'version': client_data[1], 'backend': client_data[-1], 'platform': client_data[2] if len(client_data) == 4 else None # Platform is optional } instance = ClientSubclass._get_variant(w3)(w3, **client_kwargs) return instance @property def peers(self): raise NotImplementedError @property def chain_name(self) -> str: chain_inventory = LOCAL_CHAINS if self.is_local else PUBLIC_CHAINS name = chain_inventory.get(self.chain_id, UNKNOWN_DEVELOPMENT_CHAIN_ID) return name def lock_account(self, account) -> bool: if self.is_local: return True return NotImplemented def unlock_account(self, account, password, duration=None) -> bool: if self.is_local: return True return NotImplemented @property def is_connected(self): return self.w3.isConnected() @property def etherbase(self) -> str: return self.w3.eth.accounts[0] @property def accounts(self): return self.w3.eth.accounts def get_balance(self, account): return self.w3.eth.getBalance(account) def inject_middleware(self, middleware, **kwargs): self.w3.middleware_onion.inject(middleware, **kwargs) def add_middleware(self, middleware): self.w3.middleware_onion.add(middleware) def set_gas_strategy(self, gas_strategy): self.w3.eth.setGasPriceStrategy(gas_strategy) @property def chain_id(self) -> int: try: # from hex-str return int(self.w3.eth.chainId, 16) except TypeError: # from str return int(self.w3.eth.chainId) @property def net_version(self) -> int: return int(self.w3.net.version) def get_contract(self, **kwargs) -> Contract: return self.w3.eth.contract(**kwargs) @property def gas_price(self) -> Wei: """ Returns client's gas price. Underneath, it uses the eth_gasPrice JSON-RPC method """ return self.w3.eth.gasPrice def gas_price_for_transaction(self, transaction=None) -> Wei: """ Obtains a gas price via the current gas strategy, if any; otherwise, it resorts to the client's gas price. This method mirrors the behavior of web3._utils.transactions when building transactions. """ return self.w3.eth.generateGasPrice(transaction) or self.gas_price @property def block_number(self) -> BlockNumber: return self.w3.eth.blockNumber @property def coinbase(self) -> ChecksumAddress: return self.w3.eth.coinbase def wait_for_receipt(self, transaction_hash: str, timeout: float, confirmations: int = 0) -> TxReceipt: receipt: TxReceipt = None if confirmations: # If we're waiting for confirmations, we may as well let pass some time initially to make everything easier time.sleep(self.COOLING_TIME) # We'll keep trying to get receipts until there are enough confirmations or the timeout happens with Timeout(seconds=timeout, exception=self.TransactionTimeout) as timeout_context: while not receipt: try: receipt = self.block_until_enough_confirmations( transaction_hash=transaction_hash, timeout=timeout, confirmations=confirmations) except (self.ChainReorganizationDetected, self.NotEnoughConfirmations, TimeExhausted): timeout_context.sleep( self.BLOCK_CONFIRMATIONS_POLLING_TIME) continue else: # If not asking for confirmations, just use web3 and assume the returned receipt is final try: receipt = self.w3.eth.waitForTransactionReceipt( transaction_hash=transaction_hash, timeout=timeout, poll_latency=self.TRANSACTION_POLLING_TIME) except TimeExhausted: raise # TODO: #1504 - Handle transaction timeout return receipt def block_until_enough_confirmations(self, transaction_hash: str, timeout: float, confirmations: int) -> dict: receipt: TxReceipt = self.w3.eth.waitForTransactionReceipt( transaction_hash=transaction_hash, timeout=timeout, poll_latency=self.TRANSACTION_POLLING_TIME) preliminary_block_hash = Web3.toHex(receipt['blockHash']) tx_block_number = Web3.toInt(receipt['blockNumber']) self.log.info( f"Transaction {Web3.toHex(transaction_hash)} is preliminarily included in " f"block {preliminary_block_hash}") confirmations_timeout = self._calculate_confirmations_timeout( confirmations) confirmations_so_far = 0 with Timeout(seconds=confirmations_timeout, exception=self.NotEnoughConfirmations) as timeout_context: while confirmations_so_far < confirmations: timeout_context.sleep(self.BLOCK_CONFIRMATIONS_POLLING_TIME) self.check_transaction_is_on_chain(receipt=receipt) confirmations_so_far = self.block_number - tx_block_number self.log.info( f"We have {confirmations_so_far} confirmations. " f"Waiting for {confirmations - confirmations_so_far} more." ) return receipt @staticmethod def _calculate_confirmations_timeout(confirmations): confirmations_timeout = 3 * AVERAGE_BLOCK_TIME_IN_SECONDS * confirmations return confirmations_timeout def check_transaction_is_on_chain(self, receipt: TxReceipt) -> bool: transaction_hash = Web3.toHex(receipt['transactionHash']) try: new_receipt = self.w3.eth.getTransactionReceipt(transaction_hash) except TransactionNotFound: reorg_detected = True else: reorg_detected = receipt['blockHash'] != new_receipt['blockHash'] if reorg_detected: exception = self.ChainReorganizationDetected(receipt=receipt) self.log.info(exception.message) raise exception # TODO: Consider adding an optional param in this exception to include extra info (e.g. new block) return True def sign_transaction(self, transaction_dict: dict) -> bytes: raise NotImplementedError def get_transaction(self, transaction_hash) -> dict: return self.w3.eth.getTransaction(transaction_hash) def get_transaction_receipt(self, transaction_hash) -> Union[dict, None]: return self.w3.eth.getTransactionReceipt(transaction_hash) def get_transaction_count(self, account: str, pending: bool) -> int: block_identifier = 'pending' if pending else 'latest' return self.w3.eth.getTransactionCount(account, block_identifier) def send_transaction(self, transaction_dict: dict) -> str: return self.w3.eth.sendTransaction(transaction_dict) def send_raw_transaction(self, transaction_bytes: bytes) -> str: return self.w3.eth.sendRawTransaction(transaction_bytes) def sign_message(self, account: str, message: bytes) -> str: """ Calls the appropriate signing function for the specified account on the backend. If the backend is based on eth-tester, then it uses the eth-tester signing interface to do so. """ return self.w3.eth.sign(account, data=message) def get_blocktime(self): highest_block = self.w3.eth.getBlock('latest') now = highest_block['timestamp'] return now def _has_latest_block(self) -> bool: # TODO: Investigate using `web3.middleware.make_stalecheck_middleware` #2060 # check that our local chain data is up to date return (time.time() - self.get_blocktime()) < self.STALECHECK_ALLOWABLE_DELAY def parse_transaction_data(self, transaction): return transaction.input
class CharacterConfiguration(BaseConfiguration): """ 'Sideways Engagement' of Character classes; a reflection of input parameters. """ VERSION = 2 # bump when static payload scheme changes CHARACTER_CLASS = NotImplemented DEFAULT_CONTROLLER_PORT = NotImplemented DEFAULT_DOMAIN = NetworksInventory.DEFAULT DEFAULT_NETWORK_MIDDLEWARE = RestMiddleware TEMP_CONFIGURATION_DIR_PREFIX = 'tmp-nucypher' # When we begin to support other threshold schemes, this will be one of the concepts that makes us want a factory. #571 known_node_class = Ursula # Gas DEFAULT_GAS_STRATEGY = 'fast' def __init__( self, # Base emitter=None, config_root: str = None, filepath: str = None, # Mode dev_mode: bool = False, federated_only: bool = False, # Identity checksum_address: str = None, crypto_power: CryptoPower = None, # Keyring keyring: NucypherKeyring = None, keyring_root: str = None, # Learner learn_on_same_thread: bool = False, abort_on_learning_error: bool = False, start_learning_now: bool = True, # Network controller_port: int = None, domain: str = DEFAULT_DOMAIN, interface_signature: Signature = None, network_middleware: RestMiddleware = None, lonely: bool = False, # Node Storage known_nodes: set = None, node_storage: NodeStorage = None, reload_metadata: bool = True, save_metadata: bool = True, # Blockchain poa: bool = None, light: bool = False, sync: bool = False, provider_uri: str = None, provider_process=None, gas_strategy: Union[Callable, str] = DEFAULT_GAS_STRATEGY, signer_uri: str = None, # Registry registry: BaseContractRegistry = None, registry_filepath: str = None, # Deployed Workers worker_data: dict = None): self.log = Logger(self.__class__.__name__) UNINITIALIZED_CONFIGURATION.bool_value(False) # Identity # NOTE: NodeConfigurations can only be used with Self-Characters self.is_me = True self.checksum_address = checksum_address # Keyring self.crypto_power = crypto_power self.keyring = keyring or NO_KEYRING_ATTACHED self.keyring_root = keyring_root or UNINITIALIZED_CONFIGURATION # Contract Registry if registry and registry_filepath: if registry.filepath != registry_filepath: error = f"Inconsistent registry filepaths for '{registry.filepath}' and '{registry_filepath}'." raise ValueError(error) else: self.log.warn( f"Registry and registry filepath were both passed.") self.registry = registry or NO_BLOCKCHAIN_CONNECTION.bool_value(False) self.registry_filepath = registry_filepath or UNINITIALIZED_CONFIGURATION # Blockchain self.poa = poa self.is_light = light self.provider_uri = provider_uri or NO_BLOCKCHAIN_CONNECTION self.provider_process = provider_process or NO_BLOCKCHAIN_CONNECTION self.signer_uri = signer_uri or None # Learner self.federated_only = federated_only self.domain = domain self.learn_on_same_thread = learn_on_same_thread self.abort_on_learning_error = abort_on_learning_error self.start_learning_now = start_learning_now self.save_metadata = save_metadata self.reload_metadata = reload_metadata self.known_nodes = known_nodes or set() # handpicked self.lonely = lonely # Configuration self.__dev_mode = dev_mode self.config_file_location = filepath or UNINITIALIZED_CONFIGURATION self.config_root = UNINITIALIZED_CONFIGURATION # Deployed Workers self.worker_data = worker_data # # Federated vs. Blockchain arguments consistency # # # Federated # if self.federated_only: # Check for incompatible values blockchain_args = { 'filepath': registry_filepath, 'poa': poa, 'provider_process': provider_process, 'provider_uri': provider_uri, 'gas_strategy': gas_strategy } if any(blockchain_args.values()): bad_args = ", ".join(f"{arg}={val}" for arg, val in blockchain_args.items() if val) self.log.warn( f"Arguments {bad_args} are incompatible with federated_only. " f"Overridden with a sane default.") # Clear decentralized attributes to ensure consistency with a # federated configuration. self.poa = False self.is_light = False self.provider_uri = None self.provider_process = None self.registry_filepath = None self.gas_strategy = None # # Decentralized # else: self.gas_strategy = gas_strategy is_initialized = BlockchainInterfaceFactory.is_interface_initialized( provider_uri=self.provider_uri) if not is_initialized and provider_uri: BlockchainInterfaceFactory.initialize_interface( provider_uri=self.provider_uri, poa=self.poa, light=self.is_light, provider_process=self.provider_process, sync=sync, emitter=emitter, gas_strategy=gas_strategy) else: self.log.warn( f"Using existing blockchain interface connection ({self.provider_uri})." ) if not self.registry: # TODO: These two code blocks are untested. if not self.registry_filepath: # TODO: Registry URI (goerli://speedynet.json) :-) self.log.info(f"Fetching latest registry from source.") self.registry = InMemoryContractRegistry.from_latest_publication( network=self.domain) else: self.registry = LocalContractRegistry( filepath=self.registry_filepath) self.log.info(f"Using local registry ({self.registry}).") if dev_mode: self.__temp_dir = UNINITIALIZED_CONFIGURATION self.__setup_node_storage() self.initialize(password=DEVELOPMENT_CONFIGURATION) else: self.__temp_dir = LIVE_CONFIGURATION self.config_root = config_root or self.DEFAULT_CONFIG_ROOT self._cache_runtime_filepaths() self.__setup_node_storage(node_storage=node_storage) # Network self.controller_port = controller_port or self.DEFAULT_CONTROLLER_PORT self.network_middleware = network_middleware or self.DEFAULT_NETWORK_MIDDLEWARE( registry=self.registry) self.interface_signature = interface_signature super().__init__(filepath=self.config_file_location, config_root=self.config_root) def __call__(self, **character_kwargs): return self.produce(**character_kwargs) @classmethod def checksum_address_from_filepath(cls, filepath: str) -> str: pattern = re.compile( r''' (^\w+)- (0x{1} # Then, 0x the start of the string, exactly once [0-9a-fA-F]{40}) # Followed by exactly 40 hex chars ''', re.VERBOSE) filename = os.path.basename(filepath) match = pattern.match(filename) if match: character_name, checksum_address = match.groups() else: # Extract from default by "peeking" inside the configuration file. default_name = cls.generate_filename() if filename == default_name: checksum_address = cls.peek(filepath=filepath, field='checksum_address') ########### # TODO: Cleanup and deprecate worker_address in config files, leaving only checksum_address from nucypher.config.characters import UrsulaConfiguration if isinstance(cls, UrsulaConfiguration): federated = bool( cls.peek(filepath=filepath, field='federated_only')) if not federated: checksum_address = cls.peek(filepath=cls.filepath, field='worker_address') ########### else: raise ValueError( f"Cannot extract checksum from filepath '{filepath}'") if not is_checksum_address(checksum_address): raise RuntimeError( f"Invalid checksum address detected in configuration file at '{filepath}'." ) return checksum_address def update(self, **kwargs) -> None: """ A facility for updating existing attributes on existing configuration instances. Warning: This method allows mutation and may result in an inconsistent configuration. """ return super().update(modifier=self.checksum_address, filepath=self.config_file_location, **kwargs) @classmethod def generate(cls, password: str, *args, **kwargs): """Shortcut: Hook-up a new initial installation and write configuration file to the disk""" node_config = cls(dev_mode=False, *args, **kwargs) node_config.initialize(password=password) node_config.to_configuration_file() return node_config def cleanup(self) -> None: if self.__dev_mode: self.__temp_dir.cleanup() @property def dev_mode(self) -> bool: return self.__dev_mode def __setup_node_storage(self, node_storage=None) -> None: if self.dev_mode: node_storage = ForgetfulNodeStorage( registry=self.registry, federated_only=self.federated_only) elif not node_storage: node_storage = LocalFileBasedNodeStorage( registry=self.registry, config_root=self.config_root, federated_only=self.federated_only) self.node_storage = node_storage def forget_nodes(self) -> None: self.node_storage.clear() message = "Removed all stored node node metadata and certificates" self.log.debug(message) def destroy(self) -> None: """Parse a node configuration and remove all associated files from the filesystem""" self.attach_keyring() self.keyring.destroy() os.remove(self.config_file_location) def generate_parameters(self, **overrides) -> dict: """ Warning: This method allows mutation and may result in an inconsistent configuration. """ merged_parameters = { **self.static_payload(), **self.dynamic_payload, **overrides } non_init_params = ('config_root', 'poa', 'light', 'provider_uri', 'registry_filepath', 'gas_strategy', 'signer_uri') character_init_params = filter(lambda t: t[0] not in non_init_params, merged_parameters.items()) return dict(character_init_params) def produce(self, **overrides) -> CHARACTER_CLASS: """Initialize a new character instance and return it.""" merged_parameters = self.generate_parameters(**overrides) character = self.CHARACTER_CLASS(**merged_parameters) return character @classmethod def assemble(cls, filepath: str = None, **overrides) -> dict: """ Warning: This method allows mutation and may result in an inconsistent configuration. """ payload = cls._read_configuration_file(filepath=filepath) node_storage = cls.load_node_storage( storage_payload=payload['node_storage'], federated_only=payload['federated_only']) domain = payload['domain'] # Assemble payload.update(dict(node_storage=node_storage, domain=domain)) # Filter out None values from **overrides to detect, well, overrides... # Acts as a shim for optional CLI flags. overrides = {k: v for k, v in overrides.items() if v is not None} payload = {**payload, **overrides} return payload @classmethod def from_configuration_file( cls, filepath: str = None, provider_process=None, **overrides # < ---- Inlet for CLI Flags ) -> 'CharacterConfiguration': """Initialize a CharacterConfiguration from a JSON file.""" filepath = filepath or cls.default_filepath() assembled_params = cls.assemble(filepath=filepath, **overrides) node_configuration = cls(filepath=filepath, provider_process=provider_process, **assembled_params) return node_configuration def validate(self) -> bool: # Top-level if not os.path.exists(self.config_root): raise self.ConfigurationError( f'No configuration directory found at {self.config_root}.') # Sub-paths filepaths = self.runtime_filepaths for field, path in filepaths.items(): if path and not os.path.exists(path): message = 'Missing configuration file or directory: {}.' if 'registry' in path: message += ' Did you mean to pass --federated-only?' raise CharacterConfiguration.InvalidConfiguration( message.format(path)) return True def static_payload(self) -> dict: """Exported static configuration values for initializing Ursula""" payload = dict( # Identity federated_only=self.federated_only, checksum_address=self.checksum_address, keyring_root=self.keyring_root, # Behavior domain=self.domain, learn_on_same_thread=self.learn_on_same_thread, abort_on_learning_error=self.abort_on_learning_error, start_learning_now=self.start_learning_now, save_metadata=self.save_metadata, node_storage=self.node_storage.payload(), lonely=self.lonely, ) # Optional values (mode) if not self.federated_only: if self.provider_uri: if not self.signer_uri: self.signer_uri = self.provider_uri payload.update( dict(provider_uri=self.provider_uri, poa=self.poa, light=self.is_light, signer_uri=self.signer_uri)) if self.registry_filepath: payload.update(dict(registry_filepath=self.registry_filepath)) # Gas Price payload.update(dict(gas_strategy=self.gas_strategy)) # Merge with base payload base_payload = super().static_payload() base_payload.update(payload) return payload @property # TODO: Graduate to a method and "derive" dynamic from static payload. def dynamic_payload(self) -> dict: """Exported dynamic configuration values for initializing Ursula""" payload = dict() if not self.federated_only: testnet = self.domain != NetworksInventory.MAINNET signer = Signer.from_signer_uri(self.signer_uri, testnet=testnet) payload.update(dict(registry=self.registry, signer=signer)) payload.update( dict(network_middleware=self.network_middleware or self.DEFAULT_NETWORK_MIDDLEWARE(), known_nodes=self.known_nodes, node_storage=self.node_storage, crypto_power_ups=self.derive_node_power_ups())) return payload def generate_filepath(self, filepath: str = None, modifier: str = None, override: bool = False) -> str: modifier = modifier or self.checksum_address filepath = super().generate_filepath(filepath=filepath, modifier=modifier, override=override) return filepath @property def runtime_filepaths(self) -> dict: filepaths = dict(config_root=self.config_root, keyring_root=self.keyring_root, registry_filepath=self.registry_filepath) return filepaths @classmethod def generate_runtime_filepaths(cls, config_root: str) -> dict: """Dynamically generate paths based on configuration root directory""" filepaths = dict(config_root=config_root, config_file_location=os.path.join( config_root, cls.generate_filename()), keyring_root=os.path.join(config_root, 'keyring')) return filepaths def _cache_runtime_filepaths(self) -> None: """Generate runtime filepaths and cache them on the config object""" filepaths = self.generate_runtime_filepaths( config_root=self.config_root) for field, filepath in filepaths.items(): if getattr(self, field) is UNINITIALIZED_CONFIGURATION: setattr(self, field, filepath) def attach_keyring(self, checksum_address: str = None, *args, **kwargs) -> None: account = checksum_address or self.checksum_address if not account: raise self.ConfigurationError( "No account specified to unlock keyring") if self.keyring is not NO_KEYRING_ATTACHED: if self.keyring.checksum_address != account: raise self.ConfigurationError( "There is already a keyring attached to this configuration." ) return self.keyring = NucypherKeyring(keyring_root=self.keyring_root, account=account, *args, **kwargs) def derive_node_power_ups(self) -> List[CryptoPowerUp]: power_ups = list() if self.is_me and not self.dev_mode: for power_class in self.CHARACTER_CLASS._default_crypto_powerups: power_up = self.keyring.derive_crypto_power(power_class) power_ups.append(power_up) return power_ups def initialize(self, password: str) -> str: """Initialize a new configuration and write installation files to disk.""" # Development if self.dev_mode: self.__temp_dir = TemporaryDirectory( prefix=self.TEMP_CONFIGURATION_DIR_PREFIX) self.config_root = self.__temp_dir.name # Persistent else: self._ensure_config_root_exists() self.write_keyring(password=password) self._cache_runtime_filepaths() self.node_storage.initialize() # Validate if not self.__dev_mode: self.validate() # Success message = "Created nucypher installation files at {}".format( self.config_root) self.log.debug(message) return self.config_root def write_keyring(self, password: str, checksum_address: str = None, **generation_kwargs) -> NucypherKeyring: if self.federated_only: checksum_address = FEDERATED_ADDRESS elif not checksum_address: # Note: It is assumed the blockchain interface is not yet connected. if self.provider_process: # Generate Geth's "datadir" if not os.path.exists(self.provider_process.data_dir): os.mkdir(self.provider_process.data_dir) # Get or create wallet address if not self.checksum_address: self.checksum_address = self.provider_process.ensure_account_exists( password=password) elif self.checksum_address not in self.provider_process.accounts( ): raise self.ConfigurationError( f'Unknown Account {self.checksum_address}') elif not self.checksum_address: raise self.ConfigurationError( f'No checksum address provided for decentralized configuration.' ) checksum_address = self.checksum_address self.keyring = NucypherKeyring.generate( password=password, keyring_root=self.keyring_root, checksum_address=checksum_address, **generation_kwargs) if self.federated_only: self.checksum_address = self.keyring.checksum_address return self.keyring @classmethod def load_node_storage(cls, storage_payload: dict, federated_only: bool): from nucypher.config.storages import NodeStorage node_storage_subclasses = { storage._name: storage for storage in NodeStorage.__subclasses__() } storage_type = storage_payload[NodeStorage._TYPE_LABEL] storage_class = node_storage_subclasses[storage_type] node_storage = storage_class.from_payload( payload=storage_payload, federated_only=federated_only) return node_storage
class AvailabilityTracker: FAST_INTERVAL = 15 # Seconds SLOW_INTERVAL = 60 * 2 SEEDING_DURATION = 60 MAXIMUM_ALONE_TIME = 120 MAXIMUM_SCORE = 10.0 # Score SAMPLE_SIZE = 1 # Ursulas SENSITIVITY = 0.5 # Threshold CHARGE_RATE = 0.9 # Measurement Multiplier class Unreachable(RuntimeError): pass class Solitary(Unreachable): message = "Cannot connect to any teacher nodes." class Lonely(Unreachable): message = "Cannot connect to enough teacher nodes." def __init__(self, ursula, enforce_loneliness: bool = True): self.log = Logger(self.__class__.__name__) self._ursula = ursula self.enforce_loneliness = enforce_loneliness self.__excuses = dict() # List of failure reasons self.__score = 10 # 10 == Perfect Score self.warnings = { 9: self.mild_warning, 7: self.medium_warning, 2: self.severe_warning, 1: self.shutdown_everything # 0 is unobtainable } self._start_time = None self.__active_measurement = False self.__task = LoopingCall(self.maintain) self.responders = set() @property def excuses(self): return self.__excuses def mild_warning(self) -> None: self.log.info( f'[UNREACHABLE NOTICE (SCORE {self.score})] This node was recently reported as unreachable.' ) def medium_warning(self) -> None: self.log.warn( f'[UNREACHABLE CAUTION (SCORE {self.score})] This node is reporting as unreachable.' f'Please check your network and firewall configuration.') def severe_warning(self) -> None: self.log.warn( f'[UNREACHABLE WARNING (SCORE {self.score})] ' f'Please check your network and firewall configuration.' f'Auto-shutdown will commence soon if the services do not become available.' ) def shutdown_everything(self, reason=None, halt_reactor=False): self.log.warn( f'[NODE IS UNREACHABLE (SCORE {self.score})] Commencing auto-shutdown sequence...' ) self._ursula.stop(halt_reactor=False) try: if reason: raise reason(reason.message) raise self.Unreachable( f'{self._ursula} is unreachable (scored {self.score}).') finally: if halt_reactor: self._halt_reactor() @staticmethod def _halt_reactor() -> None: if reactor.running: reactor.stop() def handle_measurement_errors(self, crash_on_error: bool = False, *args, **kwargs) -> None: if args: failure = args[0] cleaned_traceback = failure.getTraceback().replace( '{', '').replace('}', '') # FIXME: Amazing. self.log.warn( "Unhandled error during availability check: {}".format( cleaned_traceback)) if crash_on_error: failure.raiseException() else: # Restart on failure if not self.running: self.log.debug(f"Availability check crashed, restarting...") self.start(now=True) def status(self) -> bool: """Returns current indication of availability""" result = self.score > (self.SENSITIVITY * self.MAXIMUM_SCORE) if not result: for time, reason in self.__excuses.items(): self.log.info(f'[{time}] - {reason["error"]}') return result @property def running(self) -> bool: return self.__task.running def start(self, now: bool = False): if not self.running: self._start_time = maya.now() d = self.__task.start(interval=self.FAST_INTERVAL, now=now) d.addErrback(self.handle_measurement_errors) def stop(self) -> None: if self.running: self.__task.stop() def maintain(self) -> None: known_nodes_is_smaller_than_sample_size = len( self._ursula.known_nodes) < self.SAMPLE_SIZE # If there are no known nodes or too few known nodes, skip this round... # ... but not for longer than the maximum allotted alone time if known_nodes_is_smaller_than_sample_size: if not self._ursula.lonely and self.enforce_loneliness: now = maya.now().epoch delta = now - self._start_time.epoch if delta >= self.MAXIMUM_ALONE_TIME: self.severe_warning() reason = self.Solitary if not self._ursula.known_nodes else self.Lonely self.shutdown_everything(reason=reason) return if self.__task.interval == self.FAST_INTERVAL: now = maya.now().epoch delta = now - self._start_time.epoch if delta >= self.SEEDING_DURATION: # Slow down self.__task.interval = self.SLOW_INTERVAL return if self.__active_measurement: self.log.debug( f"Availability check already in progress - skipping this round (Score: {self.score}). " ) return # Abort else: self.log.debug( f"Continuing to measure availability (Score: {self.score}).") self.__active_measurement = True try: self.measure_sample() finally: self.__active_measurement = False delta = maya.now() - self._start_time self.log.info( f"Current availability score is {self.score} measured since {delta}" ) self.issue_warnings() def issue_warnings(self, cascade: bool = True) -> None: warnings = sorted(self.warnings.items(), key=lambda t: t[0]) for threshold, action in warnings: if self.score <= threshold: action() if not cascade: # Exit after the first active warning is issued return def sample(self, quantity: int) -> list: population = tuple(self._ursula.known_nodes.values()) ursulas = random.sample(population=population, k=quantity) return ursulas @property def score(self) -> float: return self.__score def record(self, result: bool = None, reason: dict = None) -> None: """Score the result and cache it.""" if (not result) and reason: self.__excuses[maya.now().epoch] = reason if result is None: return # Actually nevermind, dont score this one... score = int(result) + self.CHARGE_RATE * self.__score if score >= self.MAXIMUM_SCORE: self.__score = self.MAXIMUM_SCORE else: self.__score = score self.log.debug(f"Recorded new uptime score ({self.score})") def measure_sample(self, ursulas: list = None) -> None: """ Measure self-availability from a sample of Ursulas or automatically from known nodes. Handle the possibility of unreachable or invalid remote nodes in the sample. """ # TODO: Relocate? Unreachable = (*NodeSeemsToBeDown, self._ursula.NotStaking, self._ursula.network_middleware.UnexpectedResponse) if not ursulas: ursulas = self.sample(quantity=self.SAMPLE_SIZE) for ursula_or_sprout in ursulas: try: self.measure(ursula_or_sprout=ursula_or_sprout) except self._ursula.network_middleware.NotFound: # Ignore this measurement and move on because the remote node is not compatible. self.record(None, reason={ "error": "Remote node did not support 'ping' endpoint." }) except Unreachable as e: # This node is either not an Ursula, not available, does not support uptime checks, or is not staking... # ...do nothing and move on without changing the score. self.log.debug( f'{ursula_or_sprout} responded to uptime check with {e.__class__.__name__}' ) continue def measure(self, ursula_or_sprout: Union['Ursula', NodeSprout]) -> None: """Measure self-availability from a single remote node that participates uptime checks.""" try: response = self._ursula.network_middleware.check_availability( initiator=self._ursula, responder=ursula_or_sprout) except RestMiddleware.BadRequest as e: self.responders.add(ursula_or_sprout.checksum_address) self.record(False, reason=e.reason) else: # Record response self.responders.add(ursula_or_sprout.checksum_address) if response.status_code == 200: self.record(True) elif response.status_code == 400: self.record( False, reason={ 'failed': f"{ursula_or_sprout.checksum_address} reported unavailability." }) else: self.record( None, reason={ "error": f"{ursula_or_sprout.checksum_address} returned {response.status_code} from 'ping' endpoint." })
class StakeList(UserList): @validate_checksum_address def __init__(self, registry: BaseContractRegistry, checksum_address: ChecksumAddress = None, # allow for lazy setting *args, **kwargs): super().__init__(*args, **kwargs) self.log = Logger('stake-tracker') self.staking_agent = ContractAgency.get_agent(StakingEscrowAgent, registry=registry) from nucypher.blockchain.economics import EconomicsFactory self.economics = EconomicsFactory.get_economics(registry=registry) self.__initial_period = NOT_STAKING self.__terminal_period = NOT_STAKING # "load-in" Read on-chain stakes self.checksum_address = checksum_address self.__updated = None @property def updated(self) -> maya.MayaDT: return self.__updated @property def initial_period(self) -> int: return self.__initial_period @property def terminal_period(self) -> int: return self.__terminal_period @validate_checksum_address def refresh(self) -> None: """Public staking cache invalidation method""" return self.__read_stakes() def __read_stakes(self) -> None: """Rewrite the local staking cache by reading on-chain stakes""" existing_records = len(self) # Candidate replacement cache values current_period = self.staking_agent.get_current_period() onchain_stakes, initial_period, terminal_period = list(), 0, current_period # Read from blockchain stakes_reader = self.staking_agent.get_all_stakes(staker_address=self.checksum_address) inactive_substakes = [] for onchain_index, stake_info in enumerate(stakes_reader): if not stake_info: onchain_stake = EMPTY_STAKING_SLOT else: onchain_stake = Stake.from_stake_info(checksum_address=self.checksum_address, stake_info=stake_info, staking_agent=self.staking_agent, index=onchain_index, economics=self.economics) # rack the earliest terminal period if onchain_stake.first_locked_period: if onchain_stake.first_locked_period < initial_period: initial_period = onchain_stake.first_locked_period # rack the latest terminal period if onchain_stake.final_locked_period > terminal_period: terminal_period = onchain_stake.final_locked_period if onchain_stake.status().is_child(Stake.Status.INACTIVE): inactive_substakes.append(onchain_index) # Store the replacement stake onchain_stakes.append(onchain_stake) # Commit the new stake and terminal values to the cache self.data = onchain_stakes if onchain_stakes: self.__initial_period = initial_period self.__terminal_period = terminal_period changed_records = abs(existing_records - len(onchain_stakes)) self.log.debug(f"Updated {changed_records} local staking cache entries.") if inactive_substakes: self.log.debug(f"The following sub-stakes are inactive: {inactive_substakes}") # Record most recent cache update self.__updated = maya.now()
class Felix(Character, NucypherTokenActor): """ A NuCypher ERC20 faucet / Airdrop scheduler. Felix is a web application that gives NuCypher *testnet* tokens to registered addresses with a scheduled reduction of disbursement amounts, and an HTTP endpoint for handling new address registration. The main goal of Felix is to provide a source of testnet tokens for research and the development of production-ready nucypher dApps. """ _default_crypto_powerups = [SigningPower] # Intervals DISTRIBUTION_INTERVAL = 60 # seconds DISBURSEMENT_INTERVAL = 24 * 365 # only distribute tokens to the same address once each YEAR. STAGING_DELAY = 10 # seconds # Disbursement BATCH_SIZE = 10 # transactions MULTIPLIER = Decimal( '0.9') # 10% reduction of previous disbursement is 0.9 # this is not relevant until the year of time declared above, passes. MINIMUM_DISBURSEMENT = int(1e18) # NuNits (1 NU) ETHER_AIRDROP_AMOUNT = int(1e17) # Wei (.1 ether) MAX_INDIVIDUAL_REGISTRATIONS = 3 # Registration Limit # Node Discovery LEARNING_TIMEOUT = 30 # seconds _SHORT_LEARNING_DELAY = 60 # seconds _LONG_LEARNING_DELAY = 120 # seconds _ROUNDS_WITHOUT_NODES_AFTER_WHICH_TO_SLOW_DOWN = 1 # Twisted _CLOCK = reactor _AIRDROP_QUEUE = dict() class NoDatabase(RuntimeError): pass def __init__(self, db_filepath: str, rest_host: str, rest_port: int, client_password: str = None, crash_on_error: bool = False, distribute_ether: bool = True, registry: BaseContractRegistry = None, *args, **kwargs): # Character super().__init__(registry=registry, *args, **kwargs) self.log = Logger(f"felix-{self.checksum_address[-6::]}") # Network self.rest_port = rest_port self.rest_host = rest_host self.rest_app = NOT_RUNNING self.crash_on_error = crash_on_error # Database self.db_filepath = db_filepath self.db = NO_DATABASE_AVAILABLE self.db_engine = create_engine(f'sqlite:///{self.db_filepath}', convert_unicode=True) # Blockchain transacting_power = TransactingPower(password=client_password, account=self.checksum_address, signer=self.signer, cache=True) self._crypto_power.consume_power_up(transacting_power) self.token_agent = ContractAgency.get_agent(NucypherTokenAgent, registry=registry) self.blockchain = self.token_agent.blockchain self.reserved_addresses = [self.checksum_address, NULL_ADDRESS] # Update reserved addresses with deployed contracts existing_entries = list(registry.enrolled_addresses) self.reserved_addresses.extend(existing_entries) # Distribution self.__distributed = 0 # Track NU Output self.__airdrop = 0 # Track Batch self.__disbursement = 0 # Track Quantity self._distribution_task = LoopingCall(f=self.airdrop_tokens) self._distribution_task.clock = self._CLOCK self.start_time = NOT_RUNNING self.economics = EconomicsFactory.get_economics(registry=registry) self.MAXIMUM_DISBURSEMENT = self.economics.maximum_allowed_locked self.INITIAL_DISBURSEMENT = self.economics.minimum_allowed_locked * 3 # Optionally send ether with each token transaction self.distribute_ether = distribute_ether # Banner self.log.info(FELIX_BANNER.format(self.checksum_address)) def __repr__(self): class_name = self.__class__.__name__ r = f'{class_name}(checksum_address={self.checksum_address}, db_filepath={self.db_filepath})' return r def make_web_app(self): from flask import request from flask_sqlalchemy import SQLAlchemy # WSGI/Flask Service short_name = bytes(self.stamp).hex()[:6] self.rest_app = Flask(f"faucet-{short_name}", template_folder=TEMPLATES_DIR) self.rest_app.config[ 'SQLALCHEMY_DATABASE_URI'] = f'sqlite:///{self.db_filepath}' self.rest_app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD_CONTENT_LENGTH try: self.rest_app.secret_key = sha256( os.environ['NUCYPHER_FELIX_DB_SECRET'].encode()) # uses envvar except KeyError: raise OSError( "The 'NUCYPHER_FELIX_DB_SECRET' is not set. Export your application secret and try again." ) # Database self.db = SQLAlchemy(self.rest_app) # Database Tables class Recipient(self.db.Model): """ The one and only table in Felix's database; Used to track recipients and airdrop metadata. """ __tablename__ = 'recipient' id = self.db.Column(self.db.Integer, primary_key=True) address = self.db.Column(self.db.String, nullable=False) joined = self.db.Column(self.db.DateTime, nullable=False, default=datetime.utcnow) total_received = self.db.Column(self.db.String, default='0', nullable=False) last_disbursement_amount = self.db.Column(self.db.String, nullable=False, default=0) last_disbursement_time = self.db.Column(self.db.DateTime, nullable=True, default=None) is_staking = self.db.Column(self.db.Boolean, nullable=False, default=False) def __repr__(self): return f'{self.__class__.__name__}(id={self.id})' self.Recipient = Recipient # Bind to outer class # Flask decorators rest_app = self.rest_app # # REST Routes # @rest_app.route("/status", methods=['GET']) def status(): with ThreadedSession(self.db_engine) as session: total_recipients = session.query(self.Recipient).count() last_recipient = session.query(self.Recipient).filter( self.Recipient.last_disbursement_time.isnot( None)).order_by('last_disbursement_time').first() last_address = last_recipient.address if last_recipient else None last_transaction_date = last_recipient.last_disbursement_time.isoformat( ) if last_recipient else None unfunded = session.query(self.Recipient).filter( self.Recipient.last_disbursement_time.is_(None)).count() return json.dumps({ "total_recipients": total_recipients, "latest_recipient": last_address, "latest_disburse_date": last_transaction_date, "unfunded_recipients": unfunded, "state": { "eth": str(self.eth_balance), "NU": str(self.token_balance), "address": self.checksum_address, "contract_address": self.token_agent.contract_address, } }) @rest_app.route("/register", methods=['POST']) def register(): """Handle new recipient registration via POST request.""" new_address = (request.form.get('address') or request.get_json().get('address')) if not new_address: return Response(response="no address was supplied", status=411) if not eth_utils.is_address(new_address): return Response( response= "an invalid ethereum address was supplied. please ensure the address is a proper checksum.", status=400) else: new_address = eth_utils.to_checksum_address(new_address) if new_address in self.reserved_addresses: return Response( response= "sorry, that address is reserved and cannot receive funds.", status=403) try: with ThreadedSession(self.db_engine) as session: existing = Recipient.query.filter_by( address=new_address).all() if len(existing) > self.MAX_INDIVIDUAL_REGISTRATIONS: # Address already exists; Abort self.log.debug( f"{new_address} is already enrolled {self.MAX_INDIVIDUAL_REGISTRATIONS} times." ) return Response( response= f"{new_address} requested too many times - Please use another address.", status=409) # Create the record recipient = Recipient(address=new_address, joined=datetime.now()) session.add(recipient) session.commit() except Exception as e: # Pass along exceptions to the logger self.log.critical(str(e)) raise else: return Response(status=200) # TODO return rest_app def create_tables(self) -> None: self.make_web_app() return self.db.create_all(app=self.rest_app) def start(self, host: str, port: int, web_services: bool = True, distribution: bool = True, crash_on_error: bool = False): self.crash_on_error = crash_on_error if self.start_time is not NOT_RUNNING: raise RuntimeError("Felix is already running.") self.start_time = maya.now() payload = {"wsgi": self.rest_app, "http_port": port} deployer = HendrixDeploy(action="start", options=payload) if distribution is True: self.start_distribution() if web_services is True: deployer.run() # <-- Blocking call (Reactor) def start_distribution(self, now: bool = True) -> bool: """Start token distribution""" self.log.info(NU_BANNER) self.log.info("Starting NU Token Distribution | START") if self.token_balance == NU.ZERO(): raise self.ActorError( f"Felix address {self.checksum_address} has 0 NU tokens.") self._distribution_task.start(interval=self.DISTRIBUTION_INTERVAL, now=now) return True def stop_distribution(self) -> bool: """Start token distribution""" self.log.info("Stopping NU Token Distribution | STOP") self._distribution_task.stop() return True def __calculate_disbursement(self, recipient) -> int: """Calculate the next reward for a recipient once the are selected for distribution""" # Initial Reward - sets the future rates if recipient.last_disbursement_time is None: amount = self.INITIAL_DISBURSEMENT # Cap reached, We'll continue to leak the minimum disbursement elif int(recipient.total_received) >= self.MAXIMUM_DISBURSEMENT: amount = self.MINIMUM_DISBURSEMENT # Calculate the next disbursement else: amount = math.ceil( int(recipient.last_disbursement_amount) * self.MULTIPLIER) if amount < self.MINIMUM_DISBURSEMENT: amount = self.MINIMUM_DISBURSEMENT return int(amount) def __transfer(self, disbursement: int, recipient_address: str) -> str: """Perform a single token transfer transaction from one account to another.""" # Re-unlock from cache self.blockchain.transacting_power.activate() self.__disbursement += 1 receipt = self.token_agent.transfer( amount=disbursement, target_address=recipient_address, sender_address=self.checksum_address) txhash = receipt['transactionHash'] if self.distribute_ether: ether = self.ETHER_AIRDROP_AMOUNT transaction = { 'to': recipient_address, 'from': self.checksum_address, 'value': ether, 'gasPrice': self.blockchain.client.gas_price_for_transaction() } transaction_dict = self.blockchain.build_payload( sender_address=self.checksum_address, payload=transaction, transaction_gas_limit=22000) _receipt = self.blockchain.sign_and_broadcast_transaction( transaction_dict=transaction_dict, transaction_name='transfer') self.log.info( f"Disbursement #{self.__disbursement} OK | NU {txhash.hex()[-6:]}" f"({str(NU(disbursement, 'NuNit'))} + {self.ETHER_AIRDROP_AMOUNT} wei) -> {recipient_address}" ) else: self.log.info( f"Disbursement #{self.__disbursement} OK" f"({str(NU(disbursement, 'NuNit'))} -> {recipient_address}") return txhash def airdrop_tokens(self): """ Calculate airdrop eligibility via faucet registration and transfer tokens to selected recipients. """ with ThreadedSession(self.db_engine) as session: population = session.query(self.Recipient).count() message = f"{population} registered faucet recipients; " \ f"Distributed {str(NU(self.__distributed, 'NuNit'))} since {self.start_time.slang_time()}." self.log.debug(message) if population == 0: return # Abort - no recipients are registered. # For filtration since = datetime.now() - timedelta(hours=self.DISBURSEMENT_INTERVAL) datetime_filter = or_(self.Recipient.last_disbursement_time <= since, self.Recipient.last_disbursement_time == None) # This must be `==` not `is` with ThreadedSession(self.db_engine) as session: candidates = session.query( self.Recipient).filter(datetime_filter).all() if not candidates: self.log.info("No eligible recipients this round.") return # Discard invalid addresses, in-depth invalid_addresses = list() def siphon_invalid_entries(candidate): address_is_valid = eth_utils.is_checksum_address(candidate.address) if not address_is_valid: invalid_addresses.append(candidate.address) return address_is_valid candidates = list(filter(siphon_invalid_entries, candidates)) if invalid_addresses: self.log.info( f"{len(invalid_addresses)} invalid entries detected. Pruning database." ) # TODO: Is this needed? - Invalid entries are rejected at the endpoint view. # Prune database of invalid records # with ThreadedSession(self.db_engine) as session: # bad_eggs = session.query(self.Recipient).filter(self.Recipient.address in invalid_addresses).all() # for egg in bad_eggs: # session.delete(egg.id) # session.commit() if not candidates: self.log.info("No eligible recipients this round.") return d = threads.deferToThread(self.__do_airdrop, candidates=candidates) self._AIRDROP_QUEUE[self.__airdrop] = d return d def __do_airdrop(self, candidates: list): self.log.info(f"Staging Airdrop #{self.__airdrop}.") # Staging staged_disbursements = [(r, self.__calculate_disbursement(recipient=r)) for r in candidates] batches = list( staged_disbursements[index:index + self.BATCH_SIZE] for index in range(0, len(staged_disbursements), self.BATCH_SIZE)) total_batches = len(batches) self.log.info("====== Staged Airdrop ======") for recipient, disbursement in staged_disbursements: self.log.info(f"{recipient.address} ... {str(disbursement)[:-18]}") self.log.info("==========================") # Staging Delay self.log.info( f"Airdrop will commence in {self.STAGING_DELAY} seconds...") if self.STAGING_DELAY > 3: time.sleep(self.STAGING_DELAY - 3) for i in range(3): time.sleep(1) self.log.info(f"NU Token airdrop starting in {3 - i} seconds...") # Slowly, in series... for batch, staged_disbursement in enumerate(batches, start=1): self.log.info(f"======= Batch #{batch} ========") for recipient, disbursement in staged_disbursement: # Perform the transfer... leaky faucet. self.__transfer(disbursement=disbursement, recipient_address=recipient.address) self.__distributed += disbursement # Update the database record recipient.last_disbursement_amount = str(disbursement) recipient.total_received = str( int(recipient.total_received) + disbursement) recipient.last_disbursement_time = datetime.now() self.db.session.add(recipient) self.db.session.commit() # end inner loop self.log.info( f"Completed Airdrop #{self.__airdrop} Batch #{batch} of {total_batches}." ) # end outer loop now = maya.now() next_interval_slang = now.add( seconds=self.DISTRIBUTION_INTERVAL).slang_time() self.log.info( f"Completed Airdrop #{self.__airdrop}; Next airdrop is {next_interval_slang}." ) del self._AIRDROP_QUEUE[self.__airdrop] self.__airdrop += 1
def test_collect_rewards_integration( click_runner, testerchain, agency_local_registry, stakeholder_configuration_file_location, blockchain_alice, blockchain_bob, random_policy_label, manual_staker, manual_worker, token_economics, policy_value): half_stake_time = 2 * token_economics.minimum_locked_periods # Test setup logger = Logger("Test-CLI") # Enter the Teacher's Logger, and current_period = 0 # State the initial period for incrementing staker_address = manual_staker worker_address = manual_worker staker = Staker(domain=TEMPORARY_DOMAIN, checksum_address=staker_address, registry=agency_local_registry) staker.refresh_stakes() # The staker is staking. assert staker.is_staking assert staker.stakes assert staker.worker_address == worker_address ursula_port = select_test_port() ursula = Ursula(is_me=True, checksum_address=staker_address, signer=Web3Signer(testerchain.client), worker_address=worker_address, registry=agency_local_registry, rest_host=LOOPBACK_ADDRESS, rest_port=ursula_port, provider_uri=TEST_PROVIDER_URI, network_middleware=MockRestMiddleware(), db_filepath=tempfile.mkdtemp(), domain=TEMPORARY_DOMAIN) MOCK_KNOWN_URSULAS_CACHE[ursula_port] = ursula assert ursula.worker_address == worker_address assert ursula.checksum_address == staker_address # Make a commitment for half the first stake duration testerchain.time_travel(periods=1) for _ in range(half_stake_time): logger.debug( f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<") ursula.commit_to_next_period() testerchain.time_travel(periods=1) current_period += 1 # Alice creates a policy and grants Bob access blockchain_alice.selection_buffer = 1 M, N = 1, 1 duration_in_periods = 3 days = (duration_in_periods - 1) * (token_economics.hours_per_period // 24) now = testerchain.w3.eth.getBlock('latest').timestamp expiration = maya.MayaDT(now).add(days=days) blockchain_policy = blockchain_alice.grant(bob=blockchain_bob, label=random_policy_label, m=M, n=N, value=policy_value, expiration=expiration, handpicked_ursulas={ursula}) # Ensure that the handpicked Ursula was selected for the policy assert ursula.checksum_address in blockchain_policy.treasure_map.destinations # Bob learns about the new staker and joins the policy blockchain_bob.start_learning_loop() blockchain_bob.remember_node(node=ursula) blockchain_bob.join_policy(random_policy_label, bytes(blockchain_alice.stamp)) # Enrico Encrypts (of course) enrico = Enrico(policy_encrypting_key=blockchain_policy.public_key, network_middleware=MockRestMiddleware()) verifying_key = blockchain_alice.stamp.as_umbral_pubkey() for index in range(half_stake_time - 5): logger.debug( f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<") ursula.commit_to_next_period() # Encrypt random_data = os.urandom(random.randrange(20, 100)) ciphertext, signature = enrico.encrypt_message(plaintext=random_data) # Decrypt cleartexts = blockchain_bob.retrieve(ciphertext, enrico=enrico, alice_verifying_key=verifying_key, label=random_policy_label) assert random_data == cleartexts[0] # Ursula Staying online and the clock advancing testerchain.time_travel(periods=1) current_period += 1 # Finish the passage of time for the first Stake for _ in range(5): # plus the extended periods from stake division logger.debug( f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<") ursula.commit_to_next_period() testerchain.time_travel(periods=1) current_period += 1 # # WHERES THE MONEY URSULA?? - Collecting Rewards # # The address the client wants Ursula to send rewards to burner_wallet = testerchain.w3.eth.account.create( INSECURE_DEVELOPMENT_PASSWORD) # The rewards wallet is initially empty, because it is freshly created assert testerchain.client.get_balance(burner_wallet.address) == 0 # Rewards will be unlocked after the # final committed period has passed (+1). logger.debug(f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<") testerchain.time_travel(periods=1) current_period += 1 logger.debug(f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<") # At least half of the tokens are unlocked (restaking was enabled for some prior periods) assert staker.locked_tokens() >= token_economics.minimum_allowed_locked # Collect Policy Fee collection_args = ('stake', 'rewards', 'withdraw', '--config-file', stakeholder_configuration_file_location, '--fees', '--no-tokens', '--staking-address', staker_address, '--withdraw-address', burner_wallet.address) result = click_runner.invoke(nucypher_cli, collection_args, input=INSECURE_DEVELOPMENT_PASSWORD, catch_exceptions=False) assert result.exit_code == 0 # Policy Fee collected_policy_fee = testerchain.client.get_balance( burner_wallet.address) expected_collection = policy_value assert collected_policy_fee == expected_collection # Finish the passage of time... once and for all # Extended periods from stake division for _ in range(9): ursula.commit_to_next_period() current_period += 1 logger.debug( f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<") testerchain.time_travel(periods=1) # # Collect Staking Reward # balance_before_collecting = staker.token_agent.get_balance( address=staker_address) collection_args = ('stake', 'rewards', 'withdraw', '--config-file', stakeholder_configuration_file_location, '--no-fees', '--tokens', '--staking-address', staker_address, '--force') result = click_runner.invoke(nucypher_cli, collection_args, input=INSECURE_DEVELOPMENT_PASSWORD, catch_exceptions=False) assert result.exit_code == 0 # The staker has withdrawn her staking rewards assert staker.token_agent.get_balance( address=staker_address) > balance_before_collecting
class CharacterConfiguration(BaseConfiguration): """ 'Sideways Engagement' of Character classes; a reflection of input parameters. """ VERSION = 4 # bump when static payload scheme changes CHARACTER_CLASS = NotImplemented DEFAULT_CONTROLLER_PORT = NotImplemented MNEMONIC_KEYSTORE = False DEFAULT_DOMAIN = NetworksInventory.DEFAULT DEFAULT_NETWORK_MIDDLEWARE = RestMiddleware TEMP_CONFIGURATION_DIR_PREFIX = 'tmp-nucypher' SIGNER_ENVVAR = None # When we begin to support other threshold schemes, # this will be one of the concepts that makes us want a factory. #571 known_node_class = Ursula # Gas DEFAULT_GAS_STRATEGY = 'fast' # Payments DEFAULT_PAYMENT_METHOD = 'SubscriptionManager' DEFAULT_PAYMENT_NETWORK = 'polygon' DEFAULT_FEDERATED_PAYMENT_METHOD = 'Free' # Fields specified here are *not* passed into the Character's constructor # and can be understood as configuration fields only. _CONFIG_FIELDS = ( 'config_root', 'poa', 'light', 'registry_filepath', 'gas_strategy', 'max_gas_price', # gwei 'signer_uri', 'keystore_path', 'payment_provider', 'payment_network') def __init__( self, # Base emitter=None, config_root: Optional[Path] = None, filepath: Optional[Path] = None, # Mode dev_mode: bool = False, federated_only: bool = False, # Identity checksum_address: str = None, crypto_power: CryptoPower = None, # Keystore keystore: Keystore = None, keystore_path: Optional[Path] = None, # Learner learn_on_same_thread: bool = False, abort_on_learning_error: bool = False, start_learning_now: bool = True, # Network controller_port: int = None, domain: str = DEFAULT_DOMAIN, network_middleware: RestMiddleware = None, lonely: bool = False, # Node Storage known_nodes: set = None, node_storage: NodeStorage = None, reload_metadata: bool = True, save_metadata: bool = True, # Blockchain poa: bool = None, light: bool = False, eth_provider_uri: str = None, gas_strategy: Union[Callable, str] = DEFAULT_GAS_STRATEGY, max_gas_price: Optional[int] = None, signer_uri: str = None, # Payments # TODO: Resolve code prefixing below, possibly with the use of nested configuration fields payment_method: str = None, payment_provider: str = None, payment_network: str = None, # Registries registry: BaseContractRegistry = None, registry_filepath: Optional[Path] = None, policy_registry: BaseContractRegistry = None, policy_registry_filepath: Optional[Path] = None, # Deployed Operators worker_data: dict = None): self.log = Logger(self.__class__.__name__) # This constant is used to signal that a path can be generated if one is not provided. UNINITIALIZED_CONFIGURATION.bool_value(False) # Identity # NOTE: NodeConfigurations can only be used with Self-Characters self.is_me = True self.checksum_address = checksum_address # Keystore self.crypto_power = crypto_power if keystore_path and not keystore: keystore = Keystore(keystore_path=keystore_path) self.__keystore = self.__keystore = keystore or NO_KEYSTORE_ATTACHED.bool_value( False) self.keystore_dir = Path( keystore.keystore_path ).parent if keystore else UNINITIALIZED_CONFIGURATION # Contract Registry if registry and registry_filepath: if registry.filepath != registry_filepath: error = f"Inconsistent registry filepaths for '{registry.filepath.absolute()}'" \ f" and '{registry_filepath.absolute()}'." raise ValueError(error) else: self.log.warn( f"Registry and registry filepath were both passed.") self.registry = registry or NO_BLOCKCHAIN_CONNECTION.bool_value(False) self.registry_filepath = registry_filepath or UNINITIALIZED_CONFIGURATION self.policy_registry = policy_registry or NO_BLOCKCHAIN_CONNECTION.bool_value( False) self.policy_registry_filepath = policy_registry_filepath or UNINITIALIZED_CONFIGURATION # Blockchain self.poa = poa self.is_light = light self.eth_provider_uri = eth_provider_uri or NO_BLOCKCHAIN_CONNECTION self.signer_uri = signer_uri or None # Learner self.federated_only = federated_only self.domain = domain self.learn_on_same_thread = learn_on_same_thread self.abort_on_learning_error = abort_on_learning_error self.start_learning_now = start_learning_now self.save_metadata = save_metadata self.reload_metadata = reload_metadata self.known_nodes = known_nodes or set() # handpicked self.lonely = lonely # Configuration self.__dev_mode = dev_mode self.config_file_location = filepath or UNINITIALIZED_CONFIGURATION self.config_root = UNINITIALIZED_CONFIGURATION # Deployed Operators self.worker_data = worker_data # # Federated vs. Blockchain arguments consistency # # # Federated # if self.federated_only: # Check for incompatible values blockchain_args = { 'filepath': registry_filepath, 'poa': poa, 'eth_provider_uri': eth_provider_uri, 'payment_provider': payment_provider, 'gas_strategy': gas_strategy, 'max_gas_price': max_gas_price } if any(blockchain_args.values()): bad_args = ", ".join(f"{arg}={val}" for arg, val in blockchain_args.items() if val) self.log.warn( f"Arguments {bad_args} are incompatible with federated_only. " f"Overridden with a sane default.") # Clear decentralized attributes to ensure consistency with a # federated configuration. self.poa = False self.is_light = False self.eth_provider_uri = None self.registry_filepath = None self.policy_registry_filepath = None self.gas_strategy = None self.max_gas_price = None # Federated Payments self.payment_method = payment_method or self.DEFAULT_FEDERATED_PAYMENT_METHOD self.payment_network = payment_network self.payment_provider = payment_provider # # Decentralized # else: self.gas_strategy = gas_strategy self.max_gas_price = max_gas_price # gwei is_initialized = BlockchainInterfaceFactory.is_interface_initialized( eth_provider_uri=self.eth_provider_uri) if not is_initialized and eth_provider_uri: BlockchainInterfaceFactory.initialize_interface( eth_provider_uri=self.eth_provider_uri, poa=self.poa, light=self.is_light, emitter=emitter, gas_strategy=self.gas_strategy, max_gas_price=self.max_gas_price) else: self.log.warn( f"Using existing blockchain interface connection ({self.eth_provider_uri})." ) if not self.registry: # TODO: These two code blocks are untested. if not self.registry_filepath: # TODO: Registry URI (goerli://speedynet.json) :-) self.log.info(f"Fetching latest registry from source.") self.registry = InMemoryContractRegistry.from_latest_publication( network=self.domain) else: self.registry = LocalContractRegistry( filepath=self.registry_filepath) self.log.info(f"Using local registry ({self.registry}).") self.testnet = self.domain != NetworksInventory.MAINNET self.signer = Signer.from_signer_uri(self.signer_uri, testnet=self.testnet) # # Onchain Payments & Policies # # FIXME: Enforce this for Ursula/Alice but not Bob? from nucypher.config.characters import BobConfiguration if not isinstance(self, BobConfiguration): # if not payment_provider: # raise self.ConfigurationError("payment provider is required.") self.payment_method = payment_method or self.DEFAULT_PAYMENT_METHOD self.payment_network = payment_network or self.DEFAULT_PAYMENT_NETWORK self.payment_provider = payment_provider or ( self.eth_provider_uri or None) # default to L1 payments # TODO: Dedupe if not self.policy_registry: if not self.policy_registry_filepath: self.log.info( f"Fetching latest policy registry from source.") self.policy_registry = InMemoryContractRegistry.from_latest_publication( network=self.payment_network) else: self.policy_registry = LocalContractRegistry( filepath=self.policy_registry_filepath) self.log.info( f"Using local policy registry ({self.policy_registry})." ) if dev_mode: self.__temp_dir = UNINITIALIZED_CONFIGURATION self._setup_node_storage() self.initialize(password=DEVELOPMENT_CONFIGURATION) else: self.__temp_dir = LIVE_CONFIGURATION self.config_root = config_root or self.DEFAULT_CONFIG_ROOT self._cache_runtime_filepaths() self._setup_node_storage(node_storage=node_storage) # Network self.controller_port = controller_port or self.DEFAULT_CONTROLLER_PORT self.network_middleware = network_middleware or self.DEFAULT_NETWORK_MIDDLEWARE( registry=self.registry) super().__init__(filepath=self.config_file_location, config_root=self.config_root) def __call__(self, **character_kwargs): return self.produce(**character_kwargs) @property def keystore(self) -> Keystore: return self.__keystore def attach_keystore(self, keystore: Keystore) -> None: self.__keystore = keystore @classmethod def checksum_address_from_filepath(cls, filepath: Path) -> str: pattern = re.compile( r''' (^\w+)- (0x{1} # Then, 0x the start of the string, exactly once [0-9a-fA-F]{40}) # Followed by exactly 40 hex chars ''', re.VERBOSE) filename = filepath.name match = pattern.match(filename) if match: character_name, checksum_address = match.groups() else: # Extract from default by "peeking" inside the configuration file. default_name = cls.generate_filename() if filename == default_name: checksum_address = cls.peek(filepath=filepath, field='checksum_address') else: raise ValueError( f"Cannot extract checksum from filepath '{filepath}'") if not is_checksum_address(checksum_address): raise RuntimeError( f"Invalid checksum address detected in configuration file at '{filepath}'." ) return checksum_address def update(self, **kwargs) -> None: """ A facility for updating existing attributes on existing configuration instances. Warning: This method allows mutation and may result in an inconsistent configuration. """ # config file should exist and we we override -> no need for modifier return super().update(filepath=self.config_file_location, **kwargs) @classmethod def generate(cls, password: str, key_material: Optional[bytes] = None, *args, **kwargs): """Shortcut: Hook-up a new initial installation and configuration.""" node_config = cls(dev_mode=False, *args, **kwargs) node_config.initialize(key_material=key_material, password=password) return node_config def cleanup(self) -> None: if self.__dev_mode: self.__temp_dir.cleanup() @property def dev_mode(self) -> bool: return self.__dev_mode def _setup_node_storage(self, node_storage=None) -> None: # TODO: Disables node metadata persistence.. # if self.dev_mode: # node_storage = ForgetfulNodeStorage(registry=self.registry, federated_only=self.federated_only) # TODO: Forcibly clears the filesystem of any stored node metadata and certificates... local_node_storage = LocalFileBasedNodeStorage( registry=self.registry, config_root=self.config_root, federated_only=self.federated_only) local_node_storage.clear() self.log.info( f'Cleared peer metadata from {local_node_storage.root_dir}') # TODO: Always sets up nodes for in-memory node metadata storage node_storage = ForgetfulNodeStorage(registry=self.registry, federated_only=self.federated_only) self.node_storage = node_storage def forget_nodes(self) -> None: self.node_storage.clear() message = "Removed all stored node node metadata and certificates" self.log.debug(message) def destroy(self) -> None: """Parse a node configuration and remove all associated files from the filesystem""" self.config_file_location.unlink() def generate_parameters(self, **overrides) -> dict: """ Warning: This method allows mutation and may result in an inconsistent configuration. """ merged_parameters = { **self.static_payload(), **self.dynamic_payload, **overrides } character_init_params = filter( lambda t: t[0] not in self._CONFIG_FIELDS, merged_parameters.items()) return dict(character_init_params) def produce(self, **overrides) -> CHARACTER_CLASS: """Initialize a new character instance and return it.""" merged_parameters = self.generate_parameters(**overrides) character = self.CHARACTER_CLASS(**merged_parameters) return character @classmethod def assemble(cls, filepath: Optional[Path] = None, **overrides) -> dict: """ Warning: This method allows mutation and may result in an inconsistent configuration. """ payload = cls._read_configuration_file(filepath=filepath) node_storage = cls.load_node_storage( storage_payload=payload['node_storage'], federated_only=payload['federated_only']) max_gas_price = payload.get('max_gas_price') # gwei if max_gas_price: max_gas_price = Decimal(max_gas_price) # Assemble payload.update( dict(node_storage=node_storage, max_gas_price=max_gas_price)) payload = cast_paths_from(cls, payload) # Filter out None values from **overrides to detect, well, overrides... # Acts as a shim for optional CLI flags. overrides = {k: v for k, v in overrides.items() if v is not None} payload = {**payload, **overrides} return payload @classmethod def from_configuration_file( cls, filepath: Optional[Path] = None, **overrides # < ---- Inlet for CLI Flags ) -> 'CharacterConfiguration': """Initialize a CharacterConfiguration from a JSON file.""" filepath = filepath or cls.default_filepath() assembled_params = cls.assemble(filepath=filepath, **overrides) node_configuration = cls(filepath=filepath, **assembled_params) return node_configuration def validate(self) -> bool: # Top-level if not self.config_root.exists(): raise self.ConfigurationError( f'No configuration directory found at {self.config_root}.') # Sub-paths filepaths = self.runtime_filepaths for field, path in filepaths.items(): if path and not path.exists(): message = 'Missing configuration file or directory: {}.' if 'registry' in path: message += ' Did you mean to pass --federated-only?' raise CharacterConfiguration.InvalidConfiguration( message.format(path)) return True def static_payload(self) -> dict: """JSON-Exported static configuration values for initializing Ursula""" keystore_path = str( self.keystore.keystore_path) if self.keystore else None payload = dict( # Identity federated_only=self.federated_only, checksum_address=self.checksum_address, keystore_path=keystore_path, # Behavior domain=self.domain, learn_on_same_thread=self.learn_on_same_thread, abort_on_learning_error=self.abort_on_learning_error, start_learning_now=self.start_learning_now, save_metadata=self.save_metadata, node_storage=self.node_storage.payload(), lonely=self.lonely, ) # Optional values (mode) if not self.federated_only: if self.eth_provider_uri: if not self.signer_uri: self.signer_uri = self.eth_provider_uri payload.update( dict(eth_provider_uri=self.eth_provider_uri, poa=self.poa, light=self.is_light, signer_uri=self.signer_uri)) if self.registry_filepath: payload.update(dict(registry_filepath=self.registry_filepath)) # Gas Price __max_price = str( self.max_gas_price) if self.max_gas_price else None payload.update( dict(gas_strategy=self.gas_strategy, max_gas_price=__max_price)) # Merge with base payload base_payload = super().static_payload() base_payload.update(payload) return payload @property def dynamic_payload(self) -> dict: """ Exported dynamic configuration values for initializing Ursula. These values are used to init a character instance but are *not* saved to the JSON configuration. """ payload = dict() if not self.federated_only: payload.update(dict(registry=self.registry, signer=self.signer)) payload.update( dict(network_middleware=self.network_middleware or self.DEFAULT_NETWORK_MIDDLEWARE(), known_nodes=self.known_nodes, node_storage=self.node_storage, keystore=self.keystore, crypto_power_ups=self.derive_node_power_ups())) return payload def generate_filepath(self, filepath: Optional[Path] = None, modifier: str = None, override: bool = False) -> Path: modifier = modifier or self.checksum_address filepath = super().generate_filepath(filepath=filepath, modifier=modifier, override=override) return filepath @property def runtime_filepaths(self) -> dict: filepaths = dict(config_root=self.config_root, keystore_dir=self.keystore_dir, registry_filepath=self.registry_filepath) return filepaths @classmethod def generate_runtime_filepaths(cls, config_root: Path) -> dict: """Dynamically generate paths based on configuration root directory""" filepaths = dict(config_root=config_root, config_file_location=config_root / cls.generate_filename(), keystore_dir=config_root / 'keystore') return filepaths def _cache_runtime_filepaths(self) -> None: """Generate runtime filepaths and cache them on the config object""" filepaths = self.generate_runtime_filepaths( config_root=self.config_root) for field, filepath in filepaths.items(): if getattr(self, field) is UNINITIALIZED_CONFIGURATION: setattr(self, field, filepath) def derive_node_power_ups(self) -> List[CryptoPowerUp]: power_ups = list() if self.is_me and not self.dev_mode: for power_class in self.CHARACTER_CLASS._default_crypto_powerups: power_up = self.keystore.derive_crypto_power(power_class) power_ups.append(power_up) return power_ups def initialize(self, password: str, key_material: Optional[bytes] = None) -> str: """Initialize a new configuration and write installation files to disk.""" # Development if self.dev_mode: self.__temp_dir = TemporaryDirectory( prefix=self.TEMP_CONFIGURATION_DIR_PREFIX) self.config_root = Path(self.__temp_dir.name) # Persistent else: self._ensure_config_root_exists() self.write_keystore(key_material=key_material, password=password, interactive=self.MNEMONIC_KEYSTORE) self._cache_runtime_filepaths() self.node_storage.initialize() # Validate if not self.__dev_mode: self.validate() # Success message = "Created nucypher installation files at {}".format( self.config_root) self.log.debug(message) return Path(self.config_root) def write_keystore(self, password: str, key_material: Optional[bytes] = None, interactive: bool = True) -> Keystore: if key_material: self.__keystore = Keystore.import_secure( key_material=key_material, password=password, keystore_dir=self.keystore_dir) else: if interactive: self.__keystore = Keystore.generate( password=password, keystore_dir=self.keystore_dir, interactive=interactive) else: self.__keystore, _ = Keystore.generate( password=password, keystore_dir=self.keystore_dir, interactive=interactive) return self.keystore @classmethod def load_node_storage(cls, storage_payload: dict, federated_only: bool): from nucypher.config.storages import NodeStorage node_storage_subclasses = { storage._name: storage for storage in NodeStorage.__subclasses__() } storage_type = storage_payload[NodeStorage._TYPE_LABEL] storage_class = node_storage_subclasses[storage_type] node_storage = storage_class.from_payload( payload=storage_payload, federated_only=federated_only) return node_storage def configure_payment_method(self): # TODO: finalize config fields # Strategy-Based (current implementation, inflexible & hardcoded) # 'payment_strategy': 'SubscriptionManager' # 'payment_network': 'matic' # 'payment_provider': 'https:///matic.infura.io....' # Contract-Targeted (alternative implementation, flexible & generic) # 'payment': { # 'contract': '0xdeadbeef' # 'abi': '/home/abi/sm.json' # 'function': 'isPolicyActive' # 'provider': 'https:///matic.infura.io....' # } try: payment_class = PAYMENT_METHODS[self.payment_method] except KeyError: raise KeyError(f'Unknown payment verifier "{self.payment_method}"') if payment_class.ONCHAIN: # on-chain payment strategies require a blockchain connection payment_strategy = payment_class( network=self.payment_network, eth_provider=self.payment_provider, registry=self.policy_registry) else: payment_strategy = payment_class() return payment_strategy
class RetryRequestMiddleware: """ Automatically retries rpc requests whenever a 429 status code is returned. """ def __init__(self, make_request: Callable[[RPCEndpoint, Any], RPCResponse], w3: Web3, retries: int = 3, exponential_backoff: bool = True): self.w3 = w3 self.make_request = make_request self.retries = retries self.exponential_backoff = exponential_backoff self.logger = Logger(self.__class__.__name__) def is_request_result_retry(self, result: Union[RPCResponse, Exception]) -> bool: # default retry functionality - look for 429 codes # override for additional checks if isinstance(result, HTTPError): # HTTPError 429 status_code = result.response.status_code if status_code == 429: return True elif not isinstance(result, Exception): # must be RPCResponse if 'error' in result: error = result['error'] # either instance of RPCError or str if not isinstance(error, str) and error.get('code') == 429: return True # not retry result return False def __call__(self, method, params): result = None num_iterations = 1 + self.retries # initial call and subsequent retries for i in range(num_iterations): try: response = self.make_request(method, params) except Exception as e: # type: ignore result = e else: result = response # completed request if not self.is_request_result_retry(result): if i > 0: # not initial call and retry was actually performed self.logger.debug( f'Retried rpc request completed after {i} retries') break # max retries with no completion if i == self.retries: self.logger.warn( f'RPC request retried {self.retries} times but was not completed' ) break # backoff before next call if self.exponential_backoff: time.sleep(2**(i + 1)) # exponential back-off - 2^(retry number) if isinstance(result, Exception): raise result else: # RPCResponse return result
class LocalFileBasedNodeStorage(NodeStorage): _name = 'local' __METADATA_FILENAME_TEMPLATE = '{}.node' class NoNodeMetadataFileFound(FileNotFoundError, NodeStorage.UnknownNode): pass def __init__(self, config_root: str = None, storage_root: str = None, metadata_dir: str = None, certificates_dir: str = None, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.log = Logger(self.__class__.__name__) self.root_dir = storage_root self.metadata_dir = metadata_dir self.certificates_dir = certificates_dir self._cache_storage_filepaths(config_root=config_root) @property def source(self) -> str: """Human readable source string""" return self.root_dir @staticmethod def _generate_storage_filepaths(config_root: str = None, storage_root: str = None, metadata_dir: str = None, certificates_dir: str = None): storage_root = storage_root or os.path.join( config_root or DEFAULT_CONFIG_ROOT, 'known_nodes') metadata_dir = metadata_dir or os.path.join(storage_root, 'metadata') certificates_dir = certificates_dir or os.path.join( storage_root, 'certificates') payload = { 'storage_root': storage_root, 'metadata_dir': metadata_dir, 'certificates_dir': certificates_dir } return payload def _cache_storage_filepaths(self, config_root: str = None): filepaths = self._generate_storage_filepaths( config_root=config_root, storage_root=self.root_dir, metadata_dir=self.metadata_dir, certificates_dir=self.certificates_dir) self.root_dir = filepaths['storage_root'] self.metadata_dir = filepaths['metadata_dir'] self.certificates_dir = filepaths['certificates_dir'] # # Certificates # @validate_checksum_address def __get_certificate_filename(self, checksum_address: str): return '{}.{}'.format(checksum_address, Encoding.PEM.name.lower()) def __get_certificate_filepath(self, certificate_filename: str) -> str: return os.path.join(self.certificates_dir, certificate_filename) @validate_checksum_address def generate_certificate_filepath(self, checksum_address: str) -> str: certificate_filename = self.__get_certificate_filename( checksum_address) certificate_filepath = self.__get_certificate_filepath( certificate_filename=certificate_filename) return certificate_filepath @validate_checksum_address def __read_tls_public_certificate( self, filepath: str = None, checksum_address: str = None) -> Certificate: """Deserialize an X509 certificate from a filepath""" if not bool(filepath) ^ bool(checksum_address): raise ValueError( "Either pass filepath or checksum_address; Not both.") if not filepath and checksum_address is not None: filepath = self.generate_certificate_filepath(checksum_address) try: with open(filepath, 'rb') as certificate_file: cert = x509.load_pem_x509_certificate( certificate_file.read(), backend=default_backend()) return cert except FileNotFoundError: raise FileNotFoundError( "No SSL certificate found at {}".format(filepath)) # # Metadata # @validate_checksum_address def __generate_metadata_filepath(self, checksum_address: str, metadata_dir: str = None) -> str: metadata_path = os.path.join( metadata_dir or self.metadata_dir, self.__METADATA_FILENAME_TEMPLATE.format(checksum_address)) return metadata_path def __read_metadata(self, filepath: str, federated_only: bool): from nucypher.characters.lawful import Ursula try: with open(filepath, "rb") as seed_file: seed_file.seek(0) node_bytes = self.deserializer(seed_file.read()) node = Ursula.from_bytes(node_bytes) except FileNotFoundError: raise self.UnknownNode return node def __write_metadata(self, filepath: str, node): os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, "wb") as f: f.write(self.serializer(bytes(node))) self.log.info( "Wrote new node metadata to filesystem {}".format(filepath)) return filepath # # API # def all(self, federated_only: bool, certificates_only: bool = False) -> Set[Union[Any, Certificate]]: filenames = os.listdir( self.certificates_dir if certificates_only else self.metadata_dir) self.log.info("Found {} known node metadata files at {}".format( len(filenames), self.metadata_dir)) known_certificates = set() if certificates_only: for filename in filenames: certificate = self.__read_tls_public_certificate( os.path.join(self.certificates_dir, filename)) known_certificates.add(certificate) return known_certificates else: known_nodes = set() for filename in filenames: metadata_path = os.path.join(self.metadata_dir, filename) node = self.__read_metadata( filepath=metadata_path, federated_only=federated_only) # TODO: 466 known_nodes.add(node) return known_nodes @validate_checksum_address def get(self, checksum_address: str, federated_only: bool, certificate_only: bool = False): if certificate_only is True: certificate = self.__read_tls_public_certificate( checksum_address=checksum_address) return certificate metadata_path = self.__generate_metadata_filepath( checksum_address=checksum_address) node = self.__read_metadata(filepath=metadata_path, federated_only=federated_only) # TODO: 466 return node def store_node_certificate(self, certificate: Certificate, force: bool = True): certificate_filepath = self._write_tls_certificate( certificate=certificate, force=force) return certificate_filepath def store_node_metadata(self, node, filepath: str = None) -> str: address = node.checksum_address filepath = self.__generate_metadata_filepath(checksum_address=address, metadata_dir=filepath) self.__write_metadata(filepath=filepath, node=node) return filepath def save_node(self, node, force) -> Tuple[str, str]: certificate_filepath = self.store_node_certificate( certificate=node.certificate, force=force) metadata_filepath = self.store_node_metadata(node=node) return metadata_filepath, certificate_filepath @validate_checksum_address def remove(self, checksum_address: str, metadata: bool = True, certificate: bool = True) -> None: if metadata is True: metadata_filepath = self.__generate_metadata_filepath( checksum_address=checksum_address) os.remove(metadata_filepath) self.log.debug( "Deleted {} from the filesystem".format(checksum_address)) if certificate is True: certificate_filepath = self.generate_certificate_filepath( checksum_address=checksum_address) os.remove(certificate_filepath) self.log.debug( "Deleted {} from the filesystem".format(checksum_address)) return def clear(self, metadata: bool = True, certificates: bool = True) -> None: """Forget all stored nodes and certificates""" def __destroy_dir_contents(path) -> None: try: paths_to_remove = os.listdir(path) except FileNotFoundError: return else: for file in paths_to_remove: file_path = os.path.join(path, file) if os.path.isfile(file_path): os.unlink(file_path) if metadata is True: __destroy_dir_contents(self.metadata_dir) if certificates is True: __destroy_dir_contents(self.certificates_dir) return def payload(self) -> dict: payload = { 'storage_type': self._name, 'storage_root': self.root_dir, 'metadata_dir': self.metadata_dir, 'certificates_dir': self.certificates_dir } return payload @classmethod def from_payload(cls, payload: dict, *args, **kwargs) -> 'LocalFileBasedNodeStorage': storage_type = payload[cls._TYPE_LABEL] if not storage_type == cls._name: raise cls.NodeStorageError( "Wrong storage type. got {}".format(storage_type)) del payload['storage_type'] return cls(*args, **payload, **kwargs) def initialize(self) -> bool: storage_dirs = (self.root_dir, self.metadata_dir, self.certificates_dir) for storage_dir in storage_dirs: try: os.mkdir(storage_dir, mode=0o755) except FileExistsError: message = "There are pre-existing files at {}".format( self.root_dir) self.log.info(message) except FileNotFoundError: raise self.NodeStorageError( "There is no existing configuration at {}".format( self.root_dir)) return bool( all( map(os.path.isdir, (self.root_dir, self.metadata_dir, self.certificates_dir))))
class NodeStorage(ABC): _name = NotImplemented _TYPE_LABEL = 'storage_type' NODE_SERIALIZER = binascii.hexlify NODE_DESERIALIZER = binascii.unhexlify TLS_CERTIFICATE_ENCODING = Encoding.PEM TLS_CERTIFICATE_EXTENSION = '.{}'.format( TLS_CERTIFICATE_ENCODING.name.lower()) class NodeStorageError(Exception): pass class UnknownNode(NodeStorageError): pass class InvalidNodeCertificate(RuntimeError): """Raised when a TLS certificate is not a valid Teacher certificate.""" def __init__( self, federated_only: bool, # TODO# 466 character_class=None, serializer: Callable = NODE_SERIALIZER, deserializer: Callable = NODE_DESERIALIZER, registry: BaseContractRegistry = None, ) -> None: from nucypher.characters.lawful import Ursula self.log = Logger(self.__class__.__name__) self.registry = registry self.serializer = serializer self.deserializer = deserializer self.federated_only = federated_only self.character_class = character_class or Ursula def __getitem__(self, item): return self.get(checksum_address=item, federated_only=self.federated_only) def __setitem__(self, key, value): return self.store_node_metadata(node=value) def __delitem__(self, key): self.remove(checksum_address=key) def __iter__(self): return self.all(federated_only=self.federated_only) @property @abstractmethod def source(self) -> str: """Human readable source string""" return NotImplemented def _read_common_name(self, certificate: Certificate): x509 = OpenSSL.crypto.X509.from_cryptography(certificate) subject_components = x509.get_subject().get_components() common_name_as_bytes = subject_components[0][1] common_name_from_cert = common_name_as_bytes.decode() return common_name_from_cert def _write_tls_certificate(self, certificate: Certificate, host: str = None, force: bool = True) -> str: # Read x509 = OpenSSL.crypto.X509.from_cryptography(certificate) subject_components = x509.get_subject().get_components() common_name_as_bytes = subject_components[0][1] common_name_on_certificate = common_name_as_bytes.decode() if not host: host = common_name_on_certificate try: pseudonym = certificate.subject.get_attributes_for_oid( NameOID.PSEUDONYM)[0] except IndexError: raise self.InvalidNodeCertificate( f"Missing checksum address on certificate for host '{host}'. " f"Does this certificate belong to an Ursula?") else: checksum_address = pseudonym.value if not is_checksum_address(checksum_address): raise self.InvalidNodeCertificate( "Invalid certificate wallet address encountered: {}".format( checksum_address)) # Validate # TODO: It's better for us to have checked this a while ago so that this situation is impossible. #443 if host and (host != common_name_on_certificate): raise ValueError( f"You passed a hostname ('{host}') that does not match the certificate's common name." ) certificate_filepath = self.generate_certificate_filepath( checksum_address=checksum_address) certificate_already_exists = os.path.isfile(certificate_filepath) if force is False and certificate_already_exists: raise FileExistsError( 'A TLS certificate already exists at {}.'.format( certificate_filepath)) # Write os.makedirs(os.path.dirname(certificate_filepath), exist_ok=True) with open(certificate_filepath, 'wb') as certificate_file: public_pem_bytes = certificate.public_bytes( self.TLS_CERTIFICATE_ENCODING) certificate_file.write(public_pem_bytes) self.log.debug( f"Saved TLS certificate for {checksum_address}: {certificate_filepath}" ) return certificate_filepath @abstractmethod def store_node_certificate(self, certificate: Certificate) -> str: raise NotImplementedError @abstractmethod def store_node_metadata(self, node, filepath: str = None) -> str: """Save a single node's metadata and tls certificate""" raise NotImplementedError @abstractmethod def generate_certificate_filepath(self, checksum_address: str) -> str: raise NotImplementedError @abstractmethod def payload(self) -> dict: raise NotImplementedError @classmethod @abstractmethod def from_payload(self, data: dict, *args, **kwargs) -> 'NodeStorage': """Instantiate a storage object from a dictionary""" raise NotImplementedError @abstractmethod def initialize(self): """One-time initialization steps to establish a node storage backend""" raise NotImplementedError @abstractmethod def all(self, federated_only: bool, certificates_only: bool = False) -> set: """Return s set of all stored nodes""" raise NotImplementedError @abstractmethod def get(self, checksum_address: str, federated_only: bool): """Retrieve a single stored node""" raise NotImplementedError @abstractmethod def remove(self, checksum_address: str) -> bool: """Remove a single stored node""" raise NotImplementedError @abstractmethod def clear(self) -> bool: """Remove all stored nodes""" raise NotImplementedError
class BlockchainInterface: """ Interacts with a solidity compiler and a registry in order to instantiate compiled ethereum contracts with the given web3 provider backend. """ TIMEOUT = 600 # seconds # TODO: Correlate with the gas strategy - #2070 DEFAULT_GAS_STRATEGY = 'fast' GAS_STRATEGIES = WEB3_GAS_STRATEGIES Web3 = Web3 # TODO: This is name-shadowing the actual Web3. Is this intentional? _CONTRACT_FACTORY = VersionedContract class InterfaceError(Exception): pass class NoProvider(InterfaceError): pass class UnsupportedProvider(InterfaceError): pass class ConnectionFailed(InterfaceError): pass class UnknownContract(InterfaceError): pass REASONS = { INSUFFICIENT_ETH: 'insufficient funds for gas * price + value', } class TransactionFailed(InterfaceError): IPC_CODE = -32000 def __init__(self, message: str, transaction_dict: dict, contract_function: Union[ContractFunction, ContractConstructor], *args): self.base_message = message self.name = get_transaction_name( contract_function=contract_function) self.payload = transaction_dict self.contract_function = contract_function self.failures = { BlockchainInterface.REASONS[INSUFFICIENT_ETH]: self.insufficient_eth } self.message = self.failures.get(self.base_message, self.default) super().__init__(self.message, *args) @property def default(self) -> str: sender = self.payload["from"] message = f'{self.name} from {sender[:6]}... \n' \ f'Sender balance: {prettify_eth_amount(self.get_balance())} \n' \ f'Reason: {self.base_message} \n' \ f'Transaction: {self.payload}' return message def get_balance(self): blockchain = BlockchainInterfaceFactory.get_interface() balance = blockchain.client.get_balance( account=self.payload['from']) return balance @property def insufficient_eth(self) -> str: try: transaction_fee = self.payload['gas'] * self.payload['gasPrice'] except KeyError: return self.default else: cost = transaction_fee + self.payload.get('value', 0) message = f'{self.name} from {self.payload["from"][:8]} - {self.base_message}.' \ f'Calculated cost is {prettify_eth_amount(cost)},' \ f'but sender only has {prettify_eth_amount(self.get_balance())}.' return message def __init__( self, emitter=None, # TODO # 1754 poa: bool = None, light: bool = False, provider_uri: str = NO_BLOCKCHAIN_CONNECTION, provider: BaseProvider = NO_BLOCKCHAIN_CONNECTION, gas_strategy: Optional[Union[str, Callable]] = None, max_gas_price: Optional[int] = None): """ TODO: #1502 - Move to API docs. Filesystem Configuration Node Client EVM ================ ====================== =============== ===================== =========================== Solidity Files -- SolidityCompiler - --- HTTPProvider ------ ... | | | | | | - *BlockchainInterface* -- IPCProvider ----- External EVM (geth, parity...) | | | | TestProvider ----- EthereumTester ------------- | | PyEVM (Development Chain) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Runtime Files -- --BlockchainInterface ----> Registry | | ^ | | | | | | Key Files ------ CharacterConfiguration Agent ... (Contract API) | | ^ | | | | | | | | Actor ...Blockchain-Character API) | | ^ | | | | | | Config File --- --------- Character ... (Public API) ^ | Human The Blockchain is the junction of the solidity compiler, a contract registry, and a collection of web3 network providers as a means of interfacing with the ethereum blockchain to execute or deploy contract code on the network. Compiler and Registry Usage ----------------------------- Contracts are freshly re-compiled if an instance of SolidityCompiler is passed; otherwise, The registry will read contract data saved to disk that is be used to retrieve contact address and op-codes. Optionally, A registry instance can be passed instead. Provider Usage --------------- https: // github.com / ethereum / eth - tester # available-backends * HTTP Provider - Web3 HTTP provider, typically JSON RPC 2.0 over HTTP * Websocket Provider - Web3 WS provider, typically JSON RPC 2.0 over WS, supply endpoint uri and websocket=True * IPC Provider - Web3 File based IPC provider transported over standard I/O * Custom Provider - A pre-initialized web3.py provider instance to attach to this interface """ self.log = Logger('Blockchain') self.poa = poa self.provider_uri = provider_uri self._provider = provider self.w3 = NO_BLOCKCHAIN_CONNECTION self.client = NO_BLOCKCHAIN_CONNECTION self.is_light = light # TODO: Not ready to give users total flexibility. Let's stick for the moment to known values. See #2447 if gas_strategy not in ('slow', 'medium', 'fast', 'free', None): # FIXME: What is 'None' doing here? raise ValueError(f"'{gas_strategy}' is an invalid gas strategy") self.gas_strategy = gas_strategy or self.DEFAULT_GAS_STRATEGY self.max_gas_price = max_gas_price def __repr__(self): r = '{name}({uri})'.format(name=self.__class__.__name__, uri=self.provider_uri) return r def get_blocktime(self): return self.client.get_blocktime() @property def is_connected(self) -> bool: """ https://web3py.readthedocs.io/en/stable/__provider.html#examples-using-automated-detection """ if self.client is NO_BLOCKCHAIN_CONNECTION: return False return self.client.is_connected @classmethod def get_gas_strategy(cls, gas_strategy: Union[str, Callable] = None) -> Callable: try: gas_strategy = cls.GAS_STRATEGIES[gas_strategy] except KeyError: if gas_strategy: if not callable(gas_strategy): raise ValueError( f"{gas_strategy} must be callable to be a valid gas strategy." ) else: gas_strategy = cls.GAS_STRATEGIES[cls.DEFAULT_GAS_STRATEGY] return gas_strategy def attach_middleware(self): chain_id = int(self.client.chain_id) if self.poa is None: # If POA is not set explicitly, try to autodetect from chain id self.poa = chain_id in POA_CHAINS self.log.debug( f'Ethereum chain: {self.client.chain_name} (chain_id={chain_id}, poa={self.poa})' ) # For use with Proof-Of-Authority test-blockchains if self.poa is True: self.log.debug('Injecting POA middleware at layer 0') self.client.inject_middleware(geth_poa_middleware, layer=0) self.client.add_middleware(middleware.time_based_cache_middleware) # self.client.add_middleware(middleware.latest_block_based_cache_middleware) # TODO: This line causes failed tests and nonce reuse in tests. See #2348. self.client.add_middleware(middleware.simple_cache_middleware) self.configure_gas_strategy() def configure_gas_strategy(self, gas_strategy: Optional[Callable] = None ) -> None: if gas_strategy: reported_gas_strategy = f"fixed/{gas_strategy.name}" elif isinstance(self.client, InfuraClient): gas_strategy = construct_datafeed_median_strategy( speed=self.gas_strategy) reported_gas_strategy = f"datafeed/{self.gas_strategy}" else: reported_gas_strategy = f"web3/{self.gas_strategy}" gas_strategy = self.get_gas_strategy(self.gas_strategy) configuration_message = f"Using gas strategy '{reported_gas_strategy}'" if self.max_gas_price: __price = Web3.toWei(self.max_gas_price, 'gwei') # from gwei to wei gas_strategy = max_price_gas_strategy_wrapper( gas_strategy=gas_strategy, max_gas_price_wei=__price) configuration_message += f", with a max price of {self.max_gas_price} gwei." self.client.set_gas_strategy(gas_strategy=gas_strategy) # TODO: This line must not be called prior to establishing a connection # Move it down to a lower layer, near the client. # gwei_gas_price = Web3.fromWei(self.client.gas_price_for_transaction(), 'gwei') self.log.info(configuration_message) # self.log.debug(f"Gas strategy currently reports a gas price of {gwei_gas_price} gwei.") def connect(self): provider_uri = self.provider_uri self.log.info(f"Using external Web3 Provider '{self.provider_uri}'") # Attach Provider self._attach_provider(provider=self._provider, provider_uri=provider_uri) self.log.info("Connecting to {}".format(self.provider_uri)) if self._provider is NO_BLOCKCHAIN_CONNECTION: raise self.NoProvider( "There are no configured blockchain providers") # Connect if not connected try: self.w3 = self.Web3(provider=self._provider) self.client = EthereumClient.from_w3(w3=self.w3) except requests.ConnectionError: # RPC raise self.ConnectionFailed( f'Connection Failed - {str(self.provider_uri)} - is RPC enabled?' ) except FileNotFoundError: # IPC File Protocol raise self.ConnectionFailed( f'Connection Failed - {str(self.provider_uri)} - is IPC enabled?' ) else: self.attach_middleware() return self.is_connected @property def provider(self) -> BaseProvider: return self._provider def _attach_provider(self, provider: Optional[BaseProvider] = None, provider_uri: str = None) -> None: """ https://web3py.readthedocs.io/en/latest/providers.html#providers """ if not provider_uri and not provider: raise self.NoProvider("No URI or provider instances supplied.") if provider_uri and not provider: uri_breakdown = urlparse(provider_uri) if uri_breakdown.scheme == 'tester': providers = { 'pyevm': _get_pyevm_test_provider, 'mock': _get_mock_test_provider } provider_scheme = uri_breakdown.netloc else: providers = { 'auto': _get_auto_provider, 'ipc': _get_IPC_provider, 'file': _get_IPC_provider, 'ws': _get_websocket_provider, 'wss': _get_websocket_provider, 'http': _get_HTTP_provider, 'https': _get_HTTP_provider, } provider_scheme = uri_breakdown.scheme # auto-detect for file based ipc if not provider_scheme: if os.path.exists(provider_uri): # file is available - assume ipc/file scheme provider_scheme = 'file' self.log.info( f"Auto-detected provider scheme as 'file://' for provider {provider_uri}" ) try: self._provider = providers[provider_scheme](provider_uri) except KeyError: raise self.UnsupportedProvider( f"{provider_uri} is an invalid or unsupported blockchain provider URI" ) else: self.provider_uri = provider_uri or NO_BLOCKCHAIN_CONNECTION else: self._provider = provider @classmethod def _handle_failed_transaction(cls, exception: Exception, transaction_dict: dict, contract_function: Union[ ContractFunction, ContractConstructor], logger: Logger = None) -> None: """ Re-raising error handler and context manager for transaction broadcast or build failure events at the interface layer. This method is a last line of defense against unhandled exceptions caused by transaction failures and must raise an exception. # TODO: #1504 - Additional Handling of validation failures (gas limits, invalid fields, etc.) """ response = exception.args[0] # Assume this error is formatted as an RPC response try: code = int(response['code']) message = response['message'] except Exception: # TODO: #1504 - Try even harder to determine if this is insufficient funds causing the issue, # This may be best handled at the agent or actor layer for registry and token interactions. # Worst case scenario - raise the exception held in context implicitly raise exception if code != cls.TransactionFailed.IPC_CODE: # Only handle client-specific exceptions # https://www.jsonrpc.org/specification Section 5.1 raise exception if logger: logger.critical(message) # simple context transaction_failed = cls.TransactionFailed( message=message, # rich error (best case) contract_function=contract_function, transaction_dict=transaction_dict) raise transaction_failed from exception def __log_transaction(self, transaction_dict: dict, contract_function: ContractFunction): """ Format and log a transaction dict and return the transaction name string. This method *must not* mutate the original transaction dict. """ # Do not mutate the original transaction dict tx = dict(transaction_dict).copy() # Format if tx.get('to'): tx['to'] = to_checksum_address(contract_function.address) try: tx['selector'] = contract_function.selector except AttributeError: pass tx['from'] = to_checksum_address(tx['from']) tx.update({ f: prettify_eth_amount(v) for f, v in tx.items() if f in ('gasPrice', 'value') }) payload_pprint = ', '.join("{}: {}".format(k, v) for k, v in tx.items()) # Log transaction_name = get_transaction_name( contract_function=contract_function) self.log.debug(f"[TX-{transaction_name}] | {payload_pprint}") @validate_checksum_address def build_payload( self, sender_address: str, payload: dict = None, transaction_gas_limit: int = None, use_pending_nonce: bool = True, ) -> dict: nonce = self.client.get_transaction_count(account=sender_address, pending=use_pending_nonce) base_payload = { 'chainId': int(self.client.chain_id), 'nonce': nonce, 'from': sender_address } # Aggregate if not payload: payload = {} payload.update(base_payload) # Explicit gas override - will skip gas estimation in next operation. if transaction_gas_limit: payload['gas'] = int(transaction_gas_limit) return payload @validate_checksum_address def build_contract_transaction( self, contract_function: ContractFunction, sender_address: str, payload: dict = None, transaction_gas_limit: Optional[int] = None, gas_estimation_multiplier: Optional[float] = None, use_pending_nonce: Optional[bool] = None, ) -> dict: # Sanity checks for the gas estimation multiplier if gas_estimation_multiplier is not None: if not 1 <= gas_estimation_multiplier <= 3: # TODO: Arbitrary upper bound. raise ValueError( f"The gas estimation multiplier should be a float between 1 and 3, " f"but we received {gas_estimation_multiplier}.") elif transaction_gas_limit is not None: raise ValueError( "'transaction_gas_limit' and 'gas_estimation_multiplier' can't be used together." ) payload = self.build_payload( sender_address=sender_address, payload=payload, transaction_gas_limit=transaction_gas_limit, use_pending_nonce=use_pending_nonce) self.__log_transaction(transaction_dict=payload, contract_function=contract_function) try: if 'gas' not in payload: # As web3 buildTransaction() will estimate gas with block identifier "pending" by default, # explicitly estimate gas here with block identifier 'latest' if not otherwise specified # as a pending transaction can cause gas estimation to fail, notably in case of worklock refunds. payload['gas'] = contract_function.estimateGas( payload, block_identifier='latest') transaction_dict = contract_function.buildTransaction(payload) except (TestTransactionFailed, ValidationError, ValueError) as error: # Note: Geth (1.9.15) raises ValueError in the same condition that pyevm raises ValidationError here. # Treat this condition as "Transaction Failed" during gas estimation. raise self._handle_failed_transaction( exception=error, transaction_dict=payload, contract_function=contract_function, logger=self.log) # Overestimate the transaction gas limit according to the gas estimation multiplier, if any if gas_estimation_multiplier: gas_estimation = transaction_dict['gas'] overestimation = int( math.ceil(gas_estimation * gas_estimation_multiplier)) self.log.debug( f"Gas limit for this TX was increased from {gas_estimation} to {overestimation}, " f"using a multiplier of {gas_estimation_multiplier}.") transaction_dict['gas'] = overestimation # TODO: What if we're going over the block limit? Not likely, but perhaps worth checking (NRN) return transaction_dict def sign_and_broadcast_transaction( self, transacting_power: TransactingPower, transaction_dict: TransactionDict, transaction_name: str = "", confirmations: int = 0, fire_and_forget: bool = False) -> Union[TxReceipt, HexBytes]: """ Takes a transaction dictionary, signs it with the configured signer, then broadcasts the signed transaction using the ethereum provider's eth_sendRawTransaction RPC endpoint. Optionally blocks for receipt and confirmation with 'confirmations', and 'fire_and_forget' flags. If 'fire and forget' is True this method returns the transaction hash only, without waiting for a receipt - otherwise return the transaction receipt. """ # # Setup # # TODO # 1754 - Move this to singleton - I do not approve... nor does Bogdan? if GlobalLoggerSettings._json_ipc: emitter = JSONRPCStdoutEmitter() else: emitter = StdoutEmitter() # # Sign # # TODO: Show the USD Price: https://api.coinmarketcap.com/v1/ticker/ethereum/ price = transaction_dict['gasPrice'] price_gwei = Web3.fromWei(price, 'gwei') cost_wei = price * transaction_dict['gas'] cost = Web3.fromWei(cost_wei, 'ether') if transacting_power.is_device: emitter.message( f'Confirm transaction {transaction_name} on hardware wallet... ' f'({cost} ETH @ {price_gwei} gwei)', color='yellow') signed_raw_transaction = transacting_power.sign_transaction( transaction_dict) # # Broadcast # emitter.message( f'Broadcasting {transaction_name} Transaction ({cost} ETH @ {price_gwei} gwei)', color='yellow') try: txhash = self.client.send_raw_transaction( signed_raw_transaction) # <--- BROADCAST emitter.message(f'TXHASH {txhash.hex()}', color='yellow') except (TestTransactionFailed, ValueError): raise # TODO: Unify with Transaction failed handling -- Entry point for _handle_failed_transaction else: if fire_and_forget: return txhash # # Receipt # try: # TODO: Handle block confirmation exceptions waiting_for = 'receipt' if confirmations: waiting_for = f'{confirmations} confirmations' emitter.message( f'Waiting {self.TIMEOUT} seconds for {waiting_for}', color='yellow') receipt = self.client.wait_for_receipt(txhash, timeout=self.TIMEOUT, confirmations=confirmations) except TimeExhausted: # TODO: #1504 - Handle transaction timeout raise else: self.log.debug( f"[RECEIPT-{transaction_name}] | txhash: {receipt['transactionHash'].hex()}" ) # # Confirmations # # Primary check transaction_status = receipt.get('status', UNKNOWN_TX_STATUS) if transaction_status == 0: failure = f"Transaction transmitted, but receipt returned status code 0. " \ f"Full receipt: \n {pprint.pformat(receipt, indent=2)}" raise self.InterfaceError(failure) if transaction_status is UNKNOWN_TX_STATUS: self.log.info( f"Unknown transaction status for {txhash} (receipt did not contain a status field)" ) # Secondary check tx = self.client.get_transaction(txhash) if tx["gas"] == receipt["gasUsed"]: raise self.InterfaceError( f"Transaction consumed 100% of transaction gas." f"Full receipt: \n {pprint.pformat(receipt, indent=2)}") return receipt @validate_checksum_address def send_transaction( self, contract_function: Union[ContractFunction, ContractConstructor], transacting_power: TransactingPower, payload: dict = None, transaction_gas_limit: Optional[int] = None, gas_estimation_multiplier: Optional[float] = None, confirmations: int = 0, fire_and_forget: bool = False, # do not wait for receipt. See #2385 replace: bool = False, ) -> Union[TxReceipt, HexBytes]: if fire_and_forget: if confirmations > 0: raise ValueError( "Transaction Prevented: " "Cannot use 'confirmations' and 'fire_and_forget' options together." ) use_pending_nonce = False # TODO: #2385 else: use_pending_nonce = replace # TODO: #2385 transaction = self.build_contract_transaction( contract_function=contract_function, sender_address=transacting_power.account, payload=payload, transaction_gas_limit=transaction_gas_limit, gas_estimation_multiplier=gas_estimation_multiplier, use_pending_nonce=use_pending_nonce) # Get transaction name try: transaction_name = contract_function.fn_name.upper() except AttributeError: transaction_name = 'DEPLOY' if isinstance( contract_function, ContractConstructor) else 'UNKNOWN' txhash_or_receipt = self.sign_and_broadcast_transaction( transacting_power=transacting_power, transaction_dict=transaction, transaction_name=transaction_name, confirmations=confirmations, fire_and_forget=fire_and_forget) return txhash_or_receipt def get_contract_by_name( self, registry: BaseContractRegistry, contract_name: str, contract_version: str = None, enrollment_version: Union[int, str] = None, proxy_name: str = None, use_proxy_address: bool = True) -> VersionedContract: """ Instantiate a deployed contract from registry data, and assimilate it with its proxy if it is upgradeable. """ target_contract_records = registry.search( contract_name=contract_name, contract_version=contract_version) if not target_contract_records: raise self.UnknownContract( f"No such contract records with name {contract_name}:{contract_version}." ) if proxy_name: # Lookup proxies; Search for a published proxy that targets this contract record proxy_records = registry.search(contract_name=proxy_name) results = list() for proxy_name, proxy_version, proxy_address, proxy_abi in proxy_records: proxy_contract = self.client.w3.eth.contract( abi=proxy_abi, address=proxy_address, version=proxy_version, ContractFactoryClass=self._CONTRACT_FACTORY) # Read this dispatcher's target address from the blockchain proxy_live_target_address = proxy_contract.functions.target( ).call() for target_name, target_version, target_address, target_abi in target_contract_records: if target_address == proxy_live_target_address: if use_proxy_address: triplet = (proxy_address, target_version, target_abi) else: triplet = (target_address, target_version, target_abi) else: continue results.append(triplet) if len(results) > 1: address, _version, _abi = results[0] message = "Multiple {} deployments are targeting {}".format( proxy_name, address) raise self.InterfaceError(message.format(contract_name)) else: try: selected_address, selected_version, selected_abi = results[ 0] except IndexError: raise self.UnknownContract( f"There are no Dispatcher records targeting '{contract_name}':{contract_version}" ) else: # TODO: use_proxy_address doesnt' work in this case. Should we raise if used? # NOTE: 0 must be allowed as a valid version number if len(target_contract_records) != 1: if enrollment_version is None: m = f"{len(target_contract_records)} records enrolled " \ f"for contract {contract_name}:{contract_version} " \ f"and no version index was supplied." raise self.InterfaceError(m) enrollment_version = self.__get_enrollment_version_index( name=contract_name, contract_version=contract_version, version_index=enrollment_version, enrollments=len(target_contract_records)) else: enrollment_version = -1 # default _contract_name, selected_version, selected_address, selected_abi = target_contract_records[ enrollment_version] # Create the contract from selected sources unified_contract = self.client.w3.eth.contract( abi=selected_abi, address=selected_address, version=selected_version, ContractFactoryClass=self._CONTRACT_FACTORY) return unified_contract @staticmethod def __get_enrollment_version_index(version_index: Union[int, str], enrollments: int, name: str, contract_version: str): version_names = {'latest': -1, 'earliest': 0} try: version = version_names[version_index] except KeyError: try: version = int(version_index) except ValueError: what_is_this = version_index raise ValueError( f"'{what_is_this}' is not a valid enrollment version number" ) else: if version > enrollments - 1: message = f"Version index '{version}' is larger than the number of enrollments " \ f"for {name}:{contract_version}." raise ValueError(message) return version
class BlockchainInterface: """ Interacts with a solidity compiler and a registry in order to instantiate compiled ethereum contracts with the given web3 provider backend. """ TIMEOUT = 600 # seconds # TODO: Correlate with the gas strategy - #2070 DEFAULT_GAS_STRATEGY = 'fast' GAS_STRATEGIES = {'glacial': time_based.glacial_gas_price_strategy, # 24h 'slow': time_based.slow_gas_price_strategy, # 1h 'medium': time_based.medium_gas_price_strategy, # 5m 'fast': time_based.fast_gas_price_strategy # 60s } process = NO_PROVIDER_PROCESS.bool_value(False) Web3 = Web3 _contract_factory = VersionedContract class InterfaceError(Exception): pass class NoProvider(InterfaceError): pass class UnsupportedProvider(InterfaceError): pass class ConnectionFailed(InterfaceError): pass class UnknownContract(InterfaceError): pass REASONS = { INSUFFICIENT_ETH: 'insufficient funds for gas * price + value', } class TransactionFailed(InterfaceError): IPC_CODE = -32000 # (geth) def __init__(self, message: str, transaction_dict: dict, contract_function: Union[ContractFunction, ContractConstructor], *args): self.base_message = message self.name = get_transaction_name(contract_function=contract_function) self.payload = transaction_dict self.contract_function = contract_function self.failures = { BlockchainInterface.REASONS[INSUFFICIENT_ETH]: self.insufficient_eth } self.message = self.failures.get(self.base_message, self.default) super().__init__(self.message, *args) @property def default(self) -> str: sender = self.payload["from"] message = f'{self.name} from {sender[:6]}... \n' \ f'Sender balance: {self.get_balance()} ETH \n' \ f'Reason: {self.base_message} \n' \ f'Transaction: {self.payload}' return message def get_balance(self): blockchain = BlockchainInterfaceFactory.get_interface() balance = blockchain.client.get_balance(account=self.payload['from']) return balance @property def insufficient_eth(self) -> str: gas = (self.payload.get('gas', 1) * self.payload['gasPrice']) # FIXME: If gas is not included... cost = gas + self.payload.get('value', 0) message = f'{self.payload} from {self.payload["from"][:8]} - {self.base_message}.' \ f'Calculated cost is {cost} but sender only has {self.get_balance()}.' return message def __init__(self, emitter = None, # TODO # 1754 poa: bool = None, light: bool = False, provider_process=NO_PROVIDER_PROCESS, provider_uri: str = NO_BLOCKCHAIN_CONNECTION, provider: Web3Providers = NO_BLOCKCHAIN_CONNECTION, gas_strategy: Union[str, Callable] = DEFAULT_GAS_STRATEGY): """ TODO: #1502 - Move to API docs. Filesystem Configuration Node Client EVM ================ ====================== =============== ===================== =========================== Solidity Files -- SolidityCompiler - --- HTTPProvider ------ ... | | | | | | - *BlockchainInterface* -- IPCProvider ----- External EVM (geth, parity...) | | | | TestProvider ----- EthereumTester ------------- | | PyEVM (Development Chain) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Runtime Files -- --BlockchainInterface ----> Registry | | ^ | | | | | | Key Files ------ CharacterConfiguration Agent ... (Contract API) | | ^ | | | | | | | | Actor ...Blockchain-Character API) | | ^ | | | | | | Config File --- --------- Character ... (Public API) ^ | Human The Blockchain is the junction of the solidity compiler, a contract registry, and a collection of web3 network providers as a means of interfacing with the ethereum blockchain to execute or deploy contract code on the network. Compiler and Registry Usage ----------------------------- Contracts are freshly re-compiled if an instance of SolidityCompiler is passed; otherwise, The registry will read contract data saved to disk that is be used to retrieve contact address and op-codes. Optionally, A registry instance can be passed instead. Provider Usage --------------- https: // github.com / ethereum / eth - tester # available-backends * HTTP Provider - Web3 HTTP provider, typically JSON RPC 2.0 over HTTP * Websocket Provider - Web3 WS provider, typically JSON RPC 2.0 over WS, supply endpoint uri and websocket=True * IPC Provider - Web3 File based IPC provider transported over standard I/O * Custom Provider - A pre-initialized web3.py provider instance to attach to this interface """ self.log = Logger('Blockchain') self.poa = poa self.provider_uri = provider_uri self._provider = provider self._provider_process = provider_process self.w3 = NO_BLOCKCHAIN_CONNECTION self.client = NO_BLOCKCHAIN_CONNECTION # type: EthereumClient self.transacting_power = READ_ONLY_INTERFACE self.is_light = light self.gas_strategy = self.get_gas_strategy(gas_strategy) def __repr__(self): r = '{name}({uri})'.format(name=self.__class__.__name__, uri=self.provider_uri) return r @classmethod def from_dict(cls, payload: dict, **overrides) -> 'BlockchainInterface': payload.update({k: v for k, v in overrides.items() if v is not None}) blockchain = cls(**payload) return blockchain def to_dict(self) -> dict: payload = dict(provider_uri=self.provider_uri, poa=self.poa, light=self.is_light) return payload @property def is_connected(self) -> bool: """ https://web3py.readthedocs.io/en/stable/__provider.html#examples-using-automated-detection """ if self.client is NO_BLOCKCHAIN_CONNECTION: return False return self.client.is_connected @classmethod def get_gas_strategy(cls, gas_strategy: Union[str, Callable] = None) -> Callable: try: gas_strategy = cls.GAS_STRATEGIES[gas_strategy] except KeyError: if gas_strategy: if not callable(gas_strategy): raise ValueError(f"{gas_strategy} must be callable to be a valid gas strategy.") else: gas_strategy = cls.GAS_STRATEGIES[cls.DEFAULT_GAS_STRATEGY] return gas_strategy def attach_middleware(self): chain_id = int(self.client.chain_id) if self.poa is None: # If POA is not set explicitly, try to autodetect from chain id self.poa = chain_id in POA_CHAINS self.log.debug(f'Ethereum chain: {self.client.chain_name} (chain_id={chain_id}, poa={self.poa})') # For use with Proof-Of-Authority test-blockchains if self.poa is True: self.log.debug('Injecting POA middleware at layer 0') self.client.inject_middleware(geth_poa_middleware, layer=0) # Gas Price Strategy: # Bundled web3 strategies are too expensive for Infura (it takes ~1 minute to get a price), # so we use external gas price oracles, instead (see #2139) if isinstance(self.client, InfuraClient): gas_strategy = datafeed_fallback_gas_price_strategy else: gas_strategy = self.gas_strategy self.client.set_gas_strategy(gas_strategy=gas_strategy) gwei_gas_price = Web3.fromWei(self.client.gas_price_for_transaction(), 'gwei') self.log.debug(f"Currently, our gas strategy returns a gas price of {gwei_gas_price} gwei") self.client.add_middleware(middleware.time_based_cache_middleware) self.client.add_middleware(middleware.latest_block_based_cache_middleware) self.client.add_middleware(middleware.simple_cache_middleware) def connect(self): # Spawn child process if self._provider_process: self._provider_process.start() provider_uri = self._provider_process.provider_uri(scheme='file') else: provider_uri = self.provider_uri self.log.info(f"Using external Web3 Provider '{self.provider_uri}'") # Attach Provider self._attach_provider(provider=self._provider, provider_uri=provider_uri) self.log.info("Connecting to {}".format(self.provider_uri)) if self._provider is NO_BLOCKCHAIN_CONNECTION: raise self.NoProvider("There are no configured blockchain providers") # Connect if not connected try: self.w3 = self.Web3(provider=self._provider) self.client = EthereumClient.from_w3(w3=self.w3) except requests.ConnectionError: # RPC raise self.ConnectionFailed(f'Connection Failed - {str(self.provider_uri)} - is RPC enabled?') except FileNotFoundError: # IPC File Protocol raise self.ConnectionFailed(f'Connection Failed - {str(self.provider_uri)} - is IPC enabled?') else: self.attach_middleware() return self.is_connected def sync(self, emitter=None) -> None: sync_state = self.client.sync() if emitter is not None: emitter.echo(f"Syncing: {self.client.chain_name.capitalize()}. Waiting for sync to begin.", verbosity=1) while not len(self.client.peers): emitter.echo("waiting for peers...", verbosity=1) time.sleep(5) peer_count = len(self.client.peers) emitter.echo( f"Found {'an' if peer_count == 1 else peer_count} Ethereum peer{('s' if peer_count > 1 else '')}.", verbosity=1) try: emitter.echo("Beginning sync...", verbosity=1) initial_state = next(sync_state) except StopIteration: # will occur if no syncing needs to happen emitter.echo("Local blockchain data is already synced.", verbosity=1) return prior_state = initial_state total_blocks_to_sync = int(initial_state.get('highestBlock', 0)) - int( initial_state.get('currentBlock', 0)) with click.progressbar( length=total_blocks_to_sync, label="sync progress", file=emitter.get_stream(verbosity=1) ) as bar: for syncdata in sync_state: if syncdata: blocks_accomplished = int(syncdata['currentBlock']) - int( prior_state.get('currentBlock', 0)) bar.update(blocks_accomplished) prior_state = syncdata else: try: for syncdata in sync_state: self.client.log.info(f"Syncing {syncdata['currentBlock']}/{syncdata['highestBlock']}") except TypeError: # it's already synced return return @property def provider(self) -> Union[IPCProvider, WebsocketProvider, HTTPProvider]: return self._provider def _attach_provider(self, provider: Web3Providers = None, provider_uri: str = None) -> None: """ https://web3py.readthedocs.io/en/latest/providers.html#providers """ if not provider_uri and not provider: raise self.NoProvider("No URI or provider instances supplied.") if provider_uri and not provider: uri_breakdown = urlparse(provider_uri) if uri_breakdown.scheme == 'tester': providers = { 'pyevm': _get_pyevm_test_provider, 'geth': _get_test_geth_parity_provider, 'parity-ethereum': _get_test_geth_parity_provider, 'mock': _get_mock_test_provider } provider_scheme = uri_breakdown.netloc else: providers = { 'auto': _get_auto_provider, 'infura': _get_infura_provider, 'ipc': _get_IPC_provider, 'file': _get_IPC_provider, 'ws': _get_websocket_provider, 'wss': _get_websocket_provider, 'http': _get_HTTP_provider, 'https': _get_HTTP_provider, } provider_scheme = uri_breakdown.scheme # auto-detect for file based ipc if not provider_scheme: if os.path.exists(provider_uri): # file is available - assume ipc/file scheme provider_scheme = 'file' self.log.info(f"Auto-detected provider scheme as 'file://' for provider {provider_uri}") try: self._provider = providers[provider_scheme](provider_uri) except KeyError: raise self.UnsupportedProvider(f"{provider_uri} is an invalid or unsupported blockchain provider URI") else: self.provider_uri = provider_uri or NO_BLOCKCHAIN_CONNECTION else: self._provider = provider def __transaction_failed(self, exception: Exception, transaction_dict: dict, contract_function: Union[ContractFunction, ContractConstructor] ) -> None: """ Re-raising error handler and context manager for transaction broadcast or build failure events at the interface layer. This method is a last line of defense against unhandled exceptions caused by transaction failures and must raise an exception. # TODO: #1504 - Additional Handling of validation failures (gas limits, invalid fields, etc.) """ try: # Assume this error is formatted as an IPC response code, message = exception.args[0].values() except (ValueError, IndexError, AttributeError): # TODO: #1504 - Try even harder to determine if this is insufficient funds causing the issue, # This may be best handled at the agent or actor layer for registry and token interactions. # Worst case scenario - raise the exception held in context implicitly raise exception else: if int(code) != self.TransactionFailed.IPC_CODE: # Only handle client-specific exceptions # https://www.jsonrpc.org/specification Section 5.1 raise exception self.log.critical(message) # simple context transaction_failed = self.TransactionFailed(message=message, # rich error (best case) contract_function=contract_function, transaction_dict=transaction_dict) raise transaction_failed from exception def __log_transaction(self, transaction_dict: dict, contract_function: ContractFunction): """ Format and log a transaction dict and return the transaction name string. This method *must not* mutate the original transaction dict. """ # Do not mutate the original transaction dict tx = dict(transaction_dict).copy() # Format if tx.get('to'): tx['to'] = to_checksum_address(contract_function.address) try: tx['selector'] = contract_function.selector except AttributeError: pass tx['from'] = to_checksum_address(tx['from']) tx.update({f: prettify_eth_amount(v) for f, v in tx.items() if f in ('gasPrice', 'value')}) payload_pprint = ', '.join("{}: {}".format(k, v) for k, v in tx.items()) # Log transaction_name = get_transaction_name(contract_function=contract_function) self.log.debug(f"[TX-{transaction_name}] | {payload_pprint}") @validate_checksum_address def build_payload(self, sender_address: str, payload: dict = None, transaction_gas_limit: int = None, ) -> dict: base_payload = {'chainId': int(self.client.chain_id), 'nonce': self.client.w3.eth.getTransactionCount(sender_address, 'pending'), 'from': sender_address} # Aggregate if not payload: payload = {} payload.update(base_payload) # Explicit gas override - will skip gas estimation in next operation. if transaction_gas_limit: payload['gas'] = int(transaction_gas_limit) return payload @validate_checksum_address def build_contract_transaction(self, contract_function: ContractFunction, sender_address: str, payload: dict = None, transaction_gas_limit: int = None, ) -> dict: payload = self.build_payload(sender_address=sender_address, payload=payload, transaction_gas_limit=transaction_gas_limit) self.__log_transaction(transaction_dict=payload, contract_function=contract_function) try: transaction_dict = contract_function.buildTransaction(payload) # Gas estimation occurs here except (TestTransactionFailed, ValidationError, ValueError) as error: # Note: Geth raises ValueError in the same condition that pyevm raises ValidationError here. # Treat this condition as "Transaction Failed" during gas estimation. raise self.__transaction_failed(exception=error, transaction_dict=payload, contract_function=contract_function) return transaction_dict def sign_and_broadcast_transaction(self, transaction_dict, transaction_name: str = "", confirmations: int = 0 ) -> dict: # # Setup # # TODO # 1754 - Move this to singleton - I do not approve... nor does Bogdan? if GlobalLoggerSettings._json_ipc: emitter = JSONRPCStdoutEmitter() else: emitter = StdoutEmitter() if self.transacting_power is READ_ONLY_INTERFACE: raise self.InterfaceError(str(READ_ONLY_INTERFACE)) # # Sign # # TODO: Show the USD Price: https://api.coinmarketcap.com/v1/ticker/ethereum/ price = transaction_dict['gasPrice'] price_gwei = Web3.fromWei(price, 'gwei') cost_wei = price * transaction_dict['gas'] cost = Web3.fromWei(cost_wei, 'ether') if self.transacting_power.is_device: emitter.message(f'Confirm transaction {transaction_name} on hardware wallet... ' f'({cost} ETH @ {price_gwei} gwei)', color='yellow') signed_raw_transaction = self.transacting_power.sign_transaction(transaction_dict) # # Broadcast # emitter.message(f'Broadcasting {transaction_name} Transaction ({cost} ETH @ {price_gwei} gwei)...', color='yellow') try: txhash = self.client.send_raw_transaction(signed_raw_transaction) # <--- BROADCAST except (TestTransactionFailed, ValueError) as error: raise # TODO: Unify with Transaction failed handling # # Receipt # try: # TODO: Handle block confirmation exceptions receipt = self.client.wait_for_receipt(txhash, timeout=self.TIMEOUT, confirmations=confirmations) except TimeExhausted: # TODO: #1504 - Handle transaction timeout raise else: self.log.debug(f"[RECEIPT-{transaction_name}] | txhash: {receipt['transactionHash'].hex()}") # # Confirmations # # Primary check transaction_status = receipt.get('status', UNKNOWN_TX_STATUS) if transaction_status == 0: failure = f"Transaction transmitted, but receipt returned status code 0. " \ f"Full receipt: \n {pprint.pformat(receipt, indent=2)}" raise self.InterfaceError(failure) if transaction_status is UNKNOWN_TX_STATUS: self.log.info(f"Unknown transaction status for {txhash} (receipt did not contain a status field)") # Secondary check tx = self.client.get_transaction(txhash) if tx["gas"] == receipt["gasUsed"]: raise self.InterfaceError(f"Transaction consumed 100% of transaction gas." f"Full receipt: \n {pprint.pformat(receipt, indent=2)}") return receipt def get_blocktime(self): return self.client.get_blocktime() @validate_checksum_address def send_transaction(self, contract_function: Union[ContractFunction, ContractConstructor], sender_address: str, payload: dict = None, transaction_gas_limit: int = None, confirmations: int = 0 ) -> dict: transaction = self.build_contract_transaction(contract_function=contract_function, sender_address=sender_address, payload=payload, transaction_gas_limit=transaction_gas_limit) # Get transaction name try: transaction_name = contract_function.fn_name.upper() except AttributeError: transaction_name = 'DEPLOY' if isinstance(contract_function, ContractConstructor) else 'UNKNOWN' receipt = self.sign_and_broadcast_transaction(transaction_dict=transaction, transaction_name=transaction_name, confirmations=confirmations) return receipt def get_contract_by_name(self, registry: BaseContractRegistry, contract_name: str, contract_version: str = None, enrollment_version: Union[int, str] = None, proxy_name: str = None, use_proxy_address: bool = True ) -> VersionedContract: """ Instantiate a deployed contract from registry data, and assimilate it with its proxy if it is upgradeable. """ target_contract_records = registry.search(contract_name=contract_name, contract_version=contract_version) if not target_contract_records: raise self.UnknownContract(f"No such contract records with name {contract_name}:{contract_version}.") if proxy_name: # Lookup proxies; Search for a published proxy that targets this contract record proxy_records = registry.search(contract_name=proxy_name) results = list() for proxy_name, proxy_version, proxy_address, proxy_abi in proxy_records: proxy_contract = self.client.w3.eth.contract(abi=proxy_abi, address=proxy_address, version=proxy_version, ContractFactoryClass=self._contract_factory) # Read this dispatcher's target address from the blockchain proxy_live_target_address = proxy_contract.functions.target().call() for target_name, target_version, target_address, target_abi in target_contract_records: if target_address == proxy_live_target_address: if use_proxy_address: triplet = (proxy_address, target_version, target_abi) else: triplet = (target_address, target_version, target_abi) else: continue results.append(triplet) if len(results) > 1: address, _version, _abi = results[0] message = "Multiple {} deployments are targeting {}".format(proxy_name, address) raise self.InterfaceError(message.format(contract_name)) else: try: selected_address, selected_version, selected_abi = results[0] except IndexError: raise self.UnknownContract( f"There are no Dispatcher records targeting '{contract_name}':{contract_version}") else: # TODO: use_proxy_address doesnt' work in this case. Should we raise if used? # NOTE: 0 must be allowed as a valid version number if len(target_contract_records) != 1: if enrollment_version is None: m = f"{len(target_contract_records)} records enrolled " \ f"for contract {contract_name}:{contract_version} " \ f"and no version index was supplied." raise self.InterfaceError(m) enrollment_version = self.__get_enrollment_version_index(name=contract_name, contract_version=contract_version, version_index=enrollment_version, enrollments=len(target_contract_records)) else: enrollment_version = -1 # default _contract_name, selected_version, selected_address, selected_abi = target_contract_records[enrollment_version] # Create the contract from selected sources unified_contract = self.client.w3.eth.contract(abi=selected_abi, address=selected_address, version=selected_version, ContractFactoryClass=self._contract_factory) return unified_contract @staticmethod def __get_enrollment_version_index(version_index: Union[int, str], enrollments: int, name: str, contract_version: str): version_names = {'latest': -1, 'earliest': 0} try: version = version_names[version_index] except KeyError: try: version = int(version_index) except ValueError: what_is_this = version_index raise ValueError(f"'{what_is_this}' is not a valid enrollment version number") else: if version > enrollments - 1: message = f"Version index '{version}' is larger than the number of enrollments " \ f"for {name}:{contract_version}." raise ValueError(message) return version
def test_collect_rewards_integration( click_runner, testerchain, agency_local_registry, stakeholder_configuration_file_location, blockchain_alice, blockchain_bob, random_policy_label, beneficiary, preallocation_escrow_agent, mock_allocation_registry, manual_worker, token_economics, mock_transacting_power_activation, stake_value, policy_value, policy_rate): # Disable re-staking restake_args = ('stake', 'restake', '--disable', '--config-file', stakeholder_configuration_file_location, '--allocation-filepath', MOCK_INDIVIDUAL_ALLOCATION_FILEPATH, '--force') result = click_runner.invoke(nucypher_cli, restake_args, input=INSECURE_DEVELOPMENT_PASSWORD, catch_exceptions=False) assert result.exit_code == 0 half_stake_time = token_economics.minimum_locked_periods // 2 # Test setup logger = Logger("Test-CLI") # Enter the Teacher's Logger, and current_period = 0 # State the initial period for incrementing staker_address = preallocation_escrow_agent.principal_contract.address worker_address = manual_worker # The staker is staking. stakes = StakeList(registry=agency_local_registry, checksum_address=staker_address) stakes.refresh() assert stakes staking_agent = ContractAgency.get_agent(StakingEscrowAgent, registry=agency_local_registry) assert worker_address == staking_agent.get_worker_from_staker( staker_address=staker_address) ursula_port = select_test_port() ursula = Ursula(is_me=True, checksum_address=staker_address, worker_address=worker_address, registry=agency_local_registry, rest_host='127.0.0.1', rest_port=ursula_port, start_working_now=False, network_middleware=MockRestMiddleware(), db_filepath=tempfile.mkdtemp()) MOCK_KNOWN_URSULAS_CACHE[ursula_port] = ursula assert ursula.worker_address == worker_address assert ursula.checksum_address == staker_address mock_transacting_power_activation(account=worker_address, password=INSECURE_DEVELOPMENT_PASSWORD) # Make a commitment for half the first stake duration for _ in range(half_stake_time): logger.debug( f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<") ursula.commit_to_next_period() testerchain.time_travel(periods=1) current_period += 1 # Alice creates a policy and grants Bob access blockchain_alice.selection_buffer = 1 M, N = 1, 1 days = 3 now = testerchain.w3.eth.getBlock(block_identifier='latest').timestamp expiration = maya.MayaDT(now).add(days=days - 1) blockchain_policy = blockchain_alice.grant(bob=blockchain_bob, label=random_policy_label, m=M, n=N, value=policy_value, expiration=expiration, handpicked_ursulas={ursula}) # Ensure that the handpicked Ursula was selected for the policy arrangement = list(blockchain_policy._accepted_arrangements)[0] assert arrangement.ursula == ursula # Bob learns about the new staker and joins the policy blockchain_bob.start_learning_loop() blockchain_bob.remember_node(node=ursula) blockchain_bob.join_policy(random_policy_label, bytes(blockchain_alice.stamp)) # Enrico Encrypts (of course) enrico = Enrico(policy_encrypting_key=blockchain_policy.public_key, network_middleware=MockRestMiddleware()) verifying_key = blockchain_alice.stamp.as_umbral_pubkey() for index in range(half_stake_time - 5): logger.debug( f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<") ursula.commit_to_next_period() # Encrypt random_data = os.urandom(random.randrange(20, 100)) message_kit, signature = enrico.encrypt_message(plaintext=random_data) # Decrypt cleartexts = blockchain_bob.retrieve(message_kit, enrico=enrico, alice_verifying_key=verifying_key, label=random_policy_label) assert random_data == cleartexts[0] # Ursula Staying online and the clock advancing testerchain.time_travel(periods=1) current_period += 1 # Finish the passage of time for _ in range( 5 - 1 ): # minus 1 because the first period was already committed to in test_ursula_run logger.debug( f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<") ursula.commit_to_next_period() current_period += 1 testerchain.time_travel(periods=1) # # WHERES THE MONEY URSULA?? - Collecting Rewards # balance = testerchain.client.get_balance(beneficiary) # Rewards will be unlocked after the # final committed period has passed (+1). logger.debug(f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<") testerchain.time_travel(periods=1) current_period += 1 logger.debug(f">>>>>>>>>>> TEST PERIOD {current_period} <<<<<<<<<<<<<<<<") # Since we are mocking the blockchain connection, manually consume the transacting power of the Beneficiary. mock_transacting_power_activation(account=beneficiary, password=INSECURE_DEVELOPMENT_PASSWORD) # Collect Policy Fee collection_args = ('stake', 'collect-reward', '--config-file', stakeholder_configuration_file_location, '--policy-fee', '--no-staking-reward', '--withdraw-address', beneficiary, '--allocation-filepath', MOCK_INDIVIDUAL_ALLOCATION_FILEPATH, '--force') result = click_runner.invoke(nucypher_cli, collection_args, input=INSECURE_DEVELOPMENT_PASSWORD, catch_exceptions=False) assert result.exit_code == 0 # Policy Fee collected_policy_fee = testerchain.client.get_balance(beneficiary) assert collected_policy_fee > balance # # Collect Staking Reward # token_agent = ContractAgency.get_agent(agent_class=NucypherTokenAgent, registry=agency_local_registry) balance_before_collecting = token_agent.get_balance(address=staker_address) collection_args = ('stake', 'collect-reward', '--config-file', stakeholder_configuration_file_location, '--no-policy-fee', '--staking-reward', '--allocation-filepath', MOCK_INDIVIDUAL_ALLOCATION_FILEPATH, '--force') result = click_runner.invoke(nucypher_cli, collection_args, input=INSECURE_DEVELOPMENT_PASSWORD, catch_exceptions=False) assert result.exit_code == 0 # The beneficiary has withdrawn her staking rewards, which are now in the staking contract assert token_agent.get_balance( address=staker_address) >= balance_before_collecting
class NodeStorage(ABC): _name = NotImplemented _TYPE_LABEL = 'storage_type' TLS_CERTIFICATE_ENCODING = Encoding.PEM TLS_CERTIFICATE_EXTENSION = '.{}'.format(TLS_CERTIFICATE_ENCODING.name.lower()) class NodeStorageError(Exception): pass class UnknownNode(NodeStorageError): pass def __init__(self, federated_only: bool = False, # TODO# 466 character_class=None, registry: BaseContractRegistry = None, ) -> None: from nucypher.characters.lawful import Ursula self.log = Logger(self.__class__.__name__) self.registry = registry self.federated_only = federated_only self.character_class = character_class or Ursula def __getitem__(self, item): return self.get(checksum_address=item, federated_only=self.federated_only) def __setitem__(self, key, value): return self.store_node_metadata(node=value) def __iter__(self): return self.all(federated_only=self.federated_only) @property @abstractmethod def source(self) -> str: """Human readable source string""" return NotImplemented def encode_node_bytes(self, node_bytes): return binascii.hexlify(node_bytes) def decode_node_bytes(self, encoded_node) -> bytes: return binascii.unhexlify(encoded_node) def _read_common_name(self, certificate: Certificate): x509 = OpenSSL.crypto.X509.from_cryptography(certificate) subject_components = x509.get_subject().get_components() common_name_as_bytes = subject_components[0][1] common_name_from_cert = common_name_as_bytes.decode() return common_name_from_cert def _write_tls_certificate(self, port: int, # used to avoid duplicate certs with the same IP certificate: Certificate, force: bool = True) -> Path: # Read x509 = OpenSSL.crypto.X509.from_cryptography(certificate) subject_components = x509.get_subject().get_components() common_name_as_bytes = subject_components[0][1] common_name_on_certificate = common_name_as_bytes.decode() host = common_name_on_certificate certificate_filepath = self.generate_certificate_filepath(host=host, port=port) certificate_already_exists = certificate_filepath.is_file() if force is False and certificate_already_exists: raise FileExistsError('A TLS certificate already exists at {}.'.format(certificate_filepath)) # Write certificate_filepath.parent.mkdir(parents=True, exist_ok=True) with open(certificate_filepath, 'wb') as certificate_file: public_pem_bytes = certificate.public_bytes(self.TLS_CERTIFICATE_ENCODING) certificate_file.write(public_pem_bytes) self.log.debug(f"Saved TLS certificate for {host} to {certificate_filepath}") return certificate_filepath @abstractmethod def store_node_certificate(self, certificate: Certificate, port: int) -> Path: raise NotImplementedError @abstractmethod def store_node_metadata(self, node, filepath: Optional[Path] = None) -> Path: """Save a single node's metadata and tls certificate""" raise NotImplementedError @abstractmethod def generate_certificate_filepath(self, host: str, port: int) -> Path: raise NotImplementedError @abstractmethod def payload(self) -> dict: raise NotImplementedError @classmethod @abstractmethod def from_payload(self, data: dict, *args, **kwargs) -> 'NodeStorage': """Instantiate a storage object from a dictionary""" raise NotImplementedError @abstractmethod def initialize(self): """One-time initialization steps to establish a node storage backend""" raise NotImplementedError @abstractmethod def all(self, federated_only: bool, certificates_only: bool = False) -> set: """Return s set of all stored nodes""" raise NotImplementedError @abstractmethod def get(self, checksum_address: str, federated_only: bool): """Retrieve a single stored node""" raise NotImplementedError @abstractmethod def clear(self) -> bool: """Remove all stored nodes""" raise NotImplementedError
class Porter(Learner): BANNER = r""" ______ (_____ \ _ _____) )__ ____| |_ ____ ____ | ____/ _ \ / ___) _)/ _ )/ ___) | | | |_| | | | |_( (/ /| | |_| \___/|_| \___)____)_| the Pipe for PRE Application network operations """ APP_NAME = "Porter" _SHORT_LEARNING_DELAY = 2 _LONG_LEARNING_DELAY = 30 _ROUNDS_WITHOUT_NODES_AFTER_WHICH_TO_SLOW_DOWN = 25 DEFAULT_EXECUTION_TIMEOUT = 15 # 15s DEFAULT_PORT = 9155 _interface_class = PorterInterface class UrsulaInfo(NamedTuple): """Simple object that stores relevant Ursula information resulting from sampling.""" checksum_address: ChecksumAddress uri: str encrypting_key: PublicKey def __init__(self, domain: str = None, registry: BaseContractRegistry = None, controller: bool = True, federated_only: bool = False, node_class: object = Ursula, eth_provider_uri: str = None, execution_timeout: int = DEFAULT_EXECUTION_TIMEOUT, *args, **kwargs): self.federated_only = federated_only if not self.federated_only: if not eth_provider_uri: raise ValueError( 'ETH Provider URI is required for decentralized Porter.') if not BlockchainInterfaceFactory.is_interface_initialized( eth_provider_uri=eth_provider_uri): BlockchainInterfaceFactory.initialize_interface( eth_provider_uri=eth_provider_uri) self.registry = registry or InMemoryContractRegistry.from_latest_publication( network=domain) self.application_agent = ContractAgency.get_agent( PREApplicationAgent, registry=self.registry) else: self.registry = NO_BLOCKCHAIN_CONNECTION.bool_value(False) node_class.set_federated_mode(federated_only) super().__init__(save_metadata=True, domain=domain, node_class=node_class, *args, **kwargs) self.log = Logger(self.__class__.__name__) self.execution_timeout = execution_timeout # Controller Interface self.interface = self._interface_class(porter=self) self.controller = NO_CONTROL_PROTOCOL if controller: # TODO need to understand this better - only made it analogous to what was done for characters self.make_cli_controller() self.log.info(self.BANNER) def get_ursulas( self, quantity: int, exclude_ursulas: Optional[Sequence[ChecksumAddress]] = None, include_ursulas: Optional[Sequence[ChecksumAddress]] = None ) -> List[UrsulaInfo]: reservoir = self._make_reservoir(quantity, exclude_ursulas, include_ursulas) value_factory = PrefetchStrategy(reservoir, quantity) def get_ursula_info(ursula_address) -> Porter.UrsulaInfo: if to_checksum_address(ursula_address) not in self.known_nodes: raise ValueError(f"{ursula_address} is not known") ursula_address = to_checksum_address(ursula_address) ursula = self.known_nodes[ursula_address] try: # ensure node is up and reachable self.network_middleware.ping(ursula) return Porter.UrsulaInfo( checksum_address=ursula_address, uri=f"{ursula.rest_interface.formal_uri}", encrypting_key=ursula.public_keys(DecryptingPower)) except Exception as e: self.log.debug( f"Ursula ({ursula_address}) is unreachable: {str(e)}") raise self.block_until_number_of_known_nodes_is( quantity, timeout=self.execution_timeout, learn_on_this_thread=True, eager=True) worker_pool = WorkerPool(worker=get_ursula_info, value_factory=value_factory, target_successes=quantity, timeout=self.execution_timeout, stagger_timeout=1) worker_pool.start() try: successes = worker_pool.block_until_target_successes() finally: worker_pool.cancel() # don't wait for it to stop by "joining" - too slow... ursulas_info = successes.values() return list(ursulas_info) def retrieve_cfrags( self, treasure_map: TreasureMap, retrieval_kits: Sequence[RetrievalKit], alice_verifying_key: PublicKey, bob_encrypting_key: PublicKey, bob_verifying_key: PublicKey, ) -> List[RetrievalResult]: client = RetrievalClient(self) return client.retrieve_cfrags(treasure_map, retrieval_kits, alice_verifying_key, bob_encrypting_key, bob_verifying_key) def _make_reservoir( self, quantity: int, exclude_ursulas: Optional[Sequence[ChecksumAddress]] = None, include_ursulas: Optional[Sequence[ChecksumAddress]] = None): if self.federated_only: sample_size = quantity - (len(include_ursulas) if include_ursulas else 0) if not self.block_until_number_of_known_nodes_is( sample_size, timeout=self.execution_timeout, learn_on_this_thread=True): raise ValueError("Unable to learn about sufficient Ursulas") return make_federated_staker_reservoir( known_nodes=self.known_nodes, exclude_addresses=exclude_ursulas, include_addresses=include_ursulas) else: return make_decentralized_staking_provider_reservoir( application_agent=self.application_agent, exclude_addresses=exclude_ursulas, include_addresses=include_ursulas) def make_cli_controller(self, crash_on_error: bool = False): controller = PorterCLIController(app_name=self.APP_NAME, crash_on_error=crash_on_error, interface=self.interface) self.controller = controller return controller def make_rpc_controller(self, crash_on_error: bool = False): controller = JSONRPCController(app_name=self.APP_NAME, crash_on_error=crash_on_error, interface=self.interface) self.controller = controller return controller def make_web_controller(self, crash_on_error: bool = False, htpasswd_filepath: Path = None, cors_allow_origins_list: List[str] = None): controller = WebController( app_name=self.APP_NAME, crash_on_error=crash_on_error, interface=self._interface_class(porter=self)) self.controller = controller # Register Flask Decorator porter_flask_control = controller.make_control_transport() # CORS origins if cors_allow_origins_list: try: from flask_cors import CORS except ImportError: raise ImportError( 'Porter installation is required for to specify CORS origins ' '- run "pip install nucypher[porter]" and try again.') _ = CORS(app=porter_flask_control, origins=cors_allow_origins_list) # Basic Auth if htpasswd_filepath: try: from flask_htpasswd import HtPasswdAuth except ImportError: raise ImportError( 'Porter installation is required for basic authentication ' '- run "pip install nucypher[porter]" and try again.') porter_flask_control.config['FLASK_HTPASSWD_PATH'] = str( htpasswd_filepath.absolute()) # ensure basic auth required for all endpoints porter_flask_control.config['FLASK_AUTH_ALL'] = True _ = HtPasswdAuth(app=porter_flask_control) # # Porter Control HTTP Endpoints # @porter_flask_control.route('/get_ursulas', methods=['GET']) def get_ursulas() -> Response: """Porter control endpoint for sampling Ursulas on behalf of Alice.""" response = controller(method_name='get_ursulas', control_request=request) return response @porter_flask_control.route("/revoke", methods=['POST']) def revoke(): """Porter control endpoint for off-chain revocation of a policy on behalf of Alice.""" response = controller(method_name='revoke', control_request=request) return response @porter_flask_control.route("/retrieve_cfrags", methods=['POST']) def retrieve_cfrags() -> Response: """Porter control endpoint for executing a PRE work order on behalf of Bob.""" response = controller(method_name='retrieve_cfrags', control_request=request) return response return controller
class AnalyzeGas: """ Callable twisted log observer with built-in record-keeping for gas estimation runs. """ # Logging LOG_NAME = 'estimate-gas' LOG_FILENAME = '{}.log.json'.format(LOG_NAME) OUTPUT_DIR = os.path.join(abspath(dirname(__file__)), 'results') JSON_OUTPUT_FILENAME = '{}.json'.format(LOG_NAME) _PATTERN = re.compile(r''' ^ # Anchor at the start of a string (.+) # Any character sequence longer than 1; Captured \s=\s # Space-Equal-Space (\d+) # A sequence of digits; Captured \s\|\s # Space-Slash-Space (\d+) # A sequence of digits; Captured $ # Anchor at the end of the string ''', re.VERBOSE) def __init__(self) -> None: self.log = Logger(self.__class__.__name__) self.gas_estimations = dict() if not os.path.isdir(self.OUTPUT_DIR): os.mkdir(self.OUTPUT_DIR) @provider(ILogObserver) def __call__(self, event, *args, **kwargs) -> None: if event.get('log_namespace') == self.LOG_NAME: message = event.get("log_format") matches = self._PATTERN.match(message) if not matches: self.log.debug("No match for {} with pattern {}".format(message, self._PATTERN)) return label, estimates, gas_used = matches.groups() self.paint_line(label, estimates, gas_used) self.gas_estimations[label] = int(gas_used) @staticmethod def paint_line(label: str, estimates: str, gas_used: str) -> None: print('{label} {estimates:7,} | {gas:7,}'.format( label=label.ljust(72, '.'), estimates=int(estimates), gas=int(gas_used))) def to_json_file(self) -> None: print('Saving JSON Output...') epoch_time = str(int(time.time())) timestamped_filename = '{}-{}'.format(epoch_time, self.JSON_OUTPUT_FILENAME) filepath = os.path.join(self.OUTPUT_DIR, timestamped_filename) with open(filepath, 'w') as file: file.write(json.dumps(self.gas_estimations, indent=4)) def start_collection(self) -> None: print("Starting Data Collection...") json_filepath = os.path.join(self.OUTPUT_DIR, AnalyzeGas.LOG_FILENAME) json_io = io.open(json_filepath, "w") json_observer = jsonFileLogObserver(json_io) globalLogPublisher.addObserver(json_observer) globalLogPublisher.addObserver(self)
class SolidityCompiler: __default_contract_version = 'v0.0.0' __default_contract_dir = os.path.join(dirname(abspath(__file__)), 'source') __compiled_contracts_dir = 'contracts' __zeppelin_library_dir = 'zeppelin' __aragon_library_dir = 'aragon' optimization_runs = 200 class CompilerError(Exception): pass class VersionError(Exception): pass @classmethod def default_contract_dir(cls): return cls.__default_contract_dir def __init__(self, source_dirs: List[SourceDirs] = None, ignore_solidity_check: bool = False) -> None: # Allow for optional installation from solcx.install import get_executable self.log = Logger('solidity-compiler') version = SOLIDITY_COMPILER_VERSION if not ignore_solidity_check else None self.__sol_binary_path = get_executable(version=version) if source_dirs is None or len(source_dirs) == 0: self.source_dirs = [ SourceDirs(root_source_dir=self.__default_contract_dir) ] else: self.source_dirs = source_dirs def compile(self) -> dict: interfaces = dict() for root_source_dir, other_source_dirs in self.source_dirs: if root_source_dir is None: self.log.warn("One of the root directories is None") continue raw_interfaces = self._compile(root_source_dir, other_source_dirs) for name, data in raw_interfaces.items(): # Extract contract version from docs version_search = re.search( r""" \"details\": # @dev tag in contract docs \".*? # Skip any data in the beginning of details \| # Beginning of version definition | (v # Capture version starting from symbol v \d+ # At least one digit of major version \. # Digits splitter \d+ # At least one digit of minor version \. # Digits splitter \d+ # At least one digit of patch ) # End of capturing \| # End of version definition | .*?\" # Skip any data in the end of details """, data['devdoc'], re.VERBOSE) version = version_search.group( 1) if version_search else self.__default_contract_version try: existence_data = interfaces[name] except KeyError: existence_data = dict() interfaces.update({name: existence_data}) if version not in existence_data: existence_data.update({version: data}) return interfaces def _compile(self, root_source_dir: str, other_source_dirs: [str]) -> dict: """Executes the compiler with parameters specified in the json config""" # Allow for optional installation from solcx import compile_files from solcx.exceptions import SolcError self.log.info("Using solidity compiler binary at {}".format( self.__sol_binary_path)) contracts_dir = os.path.join(root_source_dir, self.__compiled_contracts_dir) self.log.info( "Compiling solidity source files at {}".format(contracts_dir)) source_paths = set() source_walker = os.walk(top=contracts_dir, topdown=True) if other_source_dirs is not None: for source_dir in other_source_dirs: other_source_walker = os.walk(top=source_dir, topdown=True) source_walker = itertools.chain(source_walker, other_source_walker) for root, dirs, files in source_walker: for filename in files: if filename.endswith('.sol'): path = os.path.join(root, filename) source_paths.add(path) self.log.debug( "Collecting solidity source {}".format(path)) # Compile with remappings: https://github.com/ethereum/py-solc zeppelin_dir = os.path.join(root_source_dir, self.__zeppelin_library_dir) aragon_dir = os.path.join(root_source_dir, self.__aragon_library_dir) remappings = ( "contracts={}".format(contracts_dir), "zeppelin={}".format(zeppelin_dir), "aragon={}".format(aragon_dir), ) self.log.info("Compiling with import remappings {}".format( ", ".join(remappings))) optimization_runs = self.optimization_runs try: compiled_sol = compile_files(source_files=source_paths, solc_binary=self.__sol_binary_path, import_remappings=remappings, allow_paths=root_source_dir, optimize=True, optimize_runs=optimization_runs) self.log.info( "Successfully compiled {} contracts with {} optimization runs". format(len(compiled_sol), optimization_runs)) except FileNotFoundError: raise RuntimeError( "The solidity compiler is not at the specified path. " "Check that the file exists and is executable.") except PermissionError: raise RuntimeError( "The solidity compiler binary at {} is not executable. " "Check the file's permissions.".format(self.__sol_binary_path)) except SolcError: raise # Cleanup the compiled data keys interfaces = { name.split(':')[-1]: compiled_sol[name] for name in compiled_sol } return interfaces