def test_publish(self): session = Session(WebSocketAddress('localhost', 12345, 'golem')) publisher = Publisher(session) session.publish = Mock() session.is_closing = Mock() session.is_attached = Mock() session.is_attached.return_value = True # Not connected, session closing session.connected = False session.is_closing.return_value = True with self.assertNoLogs(logger, level='WARNING'): publisher.publish('alias', 1234, kw='arg') assert not session.publish.called # Not connected, session not closing session.connected = False session.is_closing.return_value = False with self.assertLogs(logger, level='WARNING'): publisher.publish('alias', 1234, kw='arg') assert not session.publish.called # Connected session.connected = True session.is_closing.return_value = False publisher.publish('alias', 1234, kw='arg') session.publish.assert_called_with('alias', 1234, kw='arg')
def start_gui(rpc_address, gui_app=None): from golem.rpc.mapping.gui import GUI_EVENT_MAP from golem.rpc.session import Client gui_app = gui_app or GUIApp(rendering=True) events = object_method_map(gui_app.logic, GUI_EVENT_MAP) session = Session(rpc_address, events=events) reactor = install_qt5_reactor() def connect(): session.connect().addCallbacks(session_ready, start_error) def session_ready(*_): core_client = Client(session, CORE_METHOD_MAP) reactor.callFromThread(gui_app.start, core_client) reactor.callWhenRunning(connect) reactor.addSystemEventTrigger('before', 'shutdown', session.disconnect) try: reactor.run() finally: if gui_app and gui_app.gui and gui_app.gui.app: gui_app.gui.app.deleteLater()
def __init__(self, datadir: str, host: str = CROSSBAR_HOST, port: int = CROSSBAR_PORT, realm: str = CROSSBAR_REALM, ssl: bool = True) -> None: address = WebSocketAddress(host, port, realm, ssl) cert_manager = CertificateManager( os.path.join(get_local_datadir(datadir), CROSSBAR_DIR)) crsb_user = cert_manager.CrossbarUsers.golemcli secret = cert_manager.get_secret(crsb_user) self.session = Session( address, crsb_user=crsb_user, crsb_user_secret=secret, )
class RPCClient: def __init__(self, datadir: str, host: str = CROSSBAR_HOST, port: int = CROSSBAR_PORT, realm: str = CROSSBAR_REALM, ssl: bool = True) -> None: address = WebSocketAddress(host, port, realm, ssl) cert_manager = CertificateManager( os.path.join(get_local_datadir(datadir), CROSSBAR_DIR)) crsb_user = cert_manager.CrossbarUsers.golemcli secret = cert_manager.get_secret(crsb_user) self.session = Session( address, crsb_user=crsb_user, crsb_user_secret=secret, ) def call(self, method, *args, on_success=lambda: None, on_error=None, **kwargs): def on_connected(_): def default_errback(error): print("Error: {}".format(error)) deferred = self.session.call(method, *args, **kwargs) deferred.addCallbacks(on_success, on_error or default_errback) deferred.addBoth(self.shutdown) def connection_error(error): sys.stderr.write( "Error connecting to Golem instance ({}): {}\n".format( self.session.address, error)) self.shutdown() return self.session.connect(auto_reconnect=False).addCallbacks( on_connected, on_error or connection_error) def shutdown(self, *_): if self.session: self.session.disconnect()
def _backend_session_started(self, *_): self.state.frontend_session = Session(self.state.router.address, events=object_method_map( self.state.frontend, MockService.events)) self.state.frontend_deferred = self.state.frontend_session.connect() self.state.frontend_deferred.addCallbacks( self._frontend_session_started, self.state.add_errors)
def _start_backend_session(self, *_): self.state.backend_session = Session(self.state.router.address, methods=object_method_map( self.state.backend, MockService.methods)) self.state.backend_deferred = self.state.backend_session.connect() self.state.backend_deferred.addCallbacks(self._backend_session_started, self.state.add_errors)
def __init__( self, cli, # pylint: disable=too-many-arguments cert_manager: CertificateManager, host: str = CROSSBAR_HOST, port: int = CROSSBAR_PORT, realm: str = CROSSBAR_REALM, ssl: bool = True) -> None: address = WebSocketAddress(host, port, realm, ssl) self.cli = cli crsb_user = cert_manager.CrossbarUsers.golemcli self.session = Session( address, crsb_user=crsb_user, crsb_user_secret=cert_manager.get_secret(crsb_user))
class WebSocketCLI(object): class NoConnection(object): def __getattribute__(self, item): raise Exception("Cannot connect to Golem instance") def __init__(self, cli, host, port, realm, ssl=False): address = WebSocketAddress(host, port, realm, ssl) self.cli = cli self.session = Session(address) def execute(self, *args, **kwargs): from twisted.internet import reactor, threads def on_connected(_): core_client = Client(self.session, CORE_METHOD_MAP) self.cli.register_client(core_client) threads.deferToThread(self.cli.execute, *args, **kwargs).addBoth(self.shutdown) def on_error(_): self.cli.register_client(WebSocketCLI.NoConnection()) self.cli.execute(*args, **kwargs) self.shutdown() def connect(): self.session.connect().addCallbacks(on_connected, on_error) reactor.callWhenRunning(connect) reactor.run() def shutdown(self, *_): from twisted.internet import reactor from twisted.internet.error import ReactorNotRunning if self.cli: self.cli.shutdown() if self.session: self.session.disconnect() try: reactor.stop() except ReactorNotRunning: pass
def test_initialization(self): address = WebSocketAddress('host', 1234, 'realm') session = Session(address) assert isinstance(session.ready, Deferred) assert isinstance(session.config, types.ComponentConfig) assert session.config.realm == u'realm' assert not session.ready.called
def _start_session(self) -> Optional[Deferred]: if not self.rpc_router: self._stop_on_error("rpc", "RPC router is not available") return None crsb_user = self.rpc_router.cert_manager.CrossbarUsers.golemapp self.rpc_session = Session( self.rpc_router.address, cert_manager=self.rpc_router.cert_manager, use_ipv6=self._config_desc.use_ipv6, crsb_user=crsb_user, crsb_user_secret=self.rpc_router.cert_manager.get_secret( crsb_user)) deferred = self.rpc_session.connect() def on_connect(*_): methods = rpc_utils.object_method_map(self) methods['sys.exposed_procedures'] = \ self.rpc_session.exposed_procedures self.rpc_session.add_procedures(methods) self._rpc_publisher = Publisher(self.rpc_session) StatusPublisher.set_publisher(self._rpc_publisher) return deferred.addCallbacks(on_connect, self._error('rpc session'))
def _router_ready(self, *_): methods = object_method_map(self.client, CORE_METHOD_MAP) self.rpc_session = Session(self.rpc_router.address, methods=methods) self.client.configure_rpc(self.rpc_session) self.rpc_session.connect().addErrback(self._rpc_error)
class Node(object): """ Simple Golem Node connecting console user interface with Client :type client golem.client.Client: """ def __init__(self, datadir=None, transaction_system=False, **config_overrides): self.default_environments = [] self.client = Client(datadir=datadir, transaction_system=transaction_system, **config_overrides) self.rpc_router = None self.rpc_session = None import logging self.logger = logging.getLogger("app") def initialize(self): self.load_environments(self.default_environments) self.client.sync() self.client.start() def load_environments(self, environments): for env in environments: env.accept_tasks = True self.client.environments_manager.add_environment(env) def connect_with_peers(self, peers): for peer in peers: self.client.connect(peer) def add_tasks(self, tasks): for task_def in tasks: task_builder = self._get_task_builder(task_def) golem_task = Task.build_task( task_builder(self.client.get_node_name(), task_def, self.client.datadir)) self.client.enqueue_new_task(golem_task) def run(self, use_rpc=False): try: from twisted.internet import reactor if use_rpc: config = self.client.config_desc reactor.callWhenRunning(self._start_rpc_server, config.rpc_address, int(config.rpc_port)) reactor.run() except Exception as ex: self.logger.error("Reactor error: {}".format(ex)) finally: self.client.quit() sys.exit(0) def _start_rpc_server(self, host, port): from twisted.internet import reactor from golem.rpc.router import CrossbarRouter self.rpc_router = CrossbarRouter(host=host, port=port, datadir=self.client.datadir) reactor.addSystemEventTrigger("before", "shutdown", self.rpc_router.stop) self.rpc_router.start(reactor, self._router_ready, self._rpc_error) def _router_ready(self, *_): methods = object_method_map(self.client, CORE_METHOD_MAP) self.rpc_session = Session(self.rpc_router.address, methods=methods) self.client.configure_rpc(self.rpc_session) self.rpc_session.connect().addErrback(self._rpc_error) def _rpc_error(self, err): self.logger.error("RPC error: {}".format(err)) def _get_task_builder(self, task_def): raise NotImplementedError
class WebSocketCLI(object): class NoConnection(object): def __getattribute__(self, item): raise Exception("Cannot connect to Golem instance") class CLIClient(ClientProxy): def _call(self, method_alias, *args, **kwargs): from twisted.internet import reactor deferred = Deferred() super_call = super()._call def in_reactor(): try: parent_deferred = super_call( method_alias, *args, **kwargs, ) except Exception as exc: # pylint: disable=broad-except deferred.errback(exc) else: parent_deferred.chainDeferred(deferred) reactor.callFromThread(in_reactor) return deferred def __init__( self, cli, # pylint: disable=too-many-arguments cert_manager: CertificateManager, host: str = CROSSBAR_HOST, port: int = CROSSBAR_PORT, realm: str = CROSSBAR_REALM, ssl: bool = True) -> None: address = WebSocketAddress(host, port, realm, ssl) self.cli = cli crsb_user = cert_manager.CrossbarUsers.golemcli self.session = Session( address, crsb_user=crsb_user, crsb_user_secret=cert_manager.get_secret(crsb_user)) def execute(self, *args, **kwargs): from twisted.internet import reactor, threads def on_ready(_): threads.deferToThread(self.cli.execute, *args, **kwargs) \ .addBoth(self.shutdown) def on_connected(_): core_client = WebSocketCLI.CLIClient(self.session) self.cli.register_client(core_client) core_client._ready.addCallback(on_ready) # noqa pylint: disable=protected-access def on_error(error): sys.stderr.write( "Error connecting to Golem instance ({}): {}\n".format( self.session.address, error)) self.cli.register_client(WebSocketCLI.NoConnection()) self.cli.execute(*args, **kwargs) self.shutdown() def connect(): self.session.connect(auto_reconnect=False).addCallbacks( on_connected, on_error) reactor.callWhenRunning(connect) reactor.run() def shutdown(self, *_): from twisted.internet import reactor from twisted.internet.error import ReactorNotRunning if self.cli: self.cli.shutdown() if self.session: self.session.disconnect() try: reactor.stop() except ReactorNotRunning: pass
def __init__(self, cli, host, port, realm, ssl=False): address = WebSocketAddress(host, port, realm, ssl) self.cli = cli self.session = Session(address)
class Node(object): """ Simple Golem Node connecting console user interface with Client :type client golem.client.Client: """ def __init__( self, # noqa pylint: disable=too-many-arguments datadir: str, app_config: AppConfig, config_desc: ClientConfigDescriptor, # SEE golem.core.variables.CONCENT_CHOICES concent_variant: dict, peers: Optional[List[SocketAddress]] = None, use_monitor: bool = None, use_talkback: bool = None, use_docker_manager: bool = True, geth_address: Optional[str] = None, password: Optional[str] = None) -> None: # DO NOT MAKE THIS IMPORT GLOBAL # otherwise, reactor will install global signal handlers on import # and will prevent the IOCP / kqueue reactors from being installed. from twisted.internet import reactor self._reactor = reactor self._app_config = app_config self._config_desc = config_desc self._datadir = datadir self._use_docker_manager = use_docker_manager self._docker_manager: Optional[DockerManager] = None self._use_monitor = config_desc.enable_monitor \ if use_monitor is None else use_monitor self._use_talkback = config_desc.enable_talkback \ if use_talkback is None else use_talkback self._keys_auth: Optional[KeysAuth] = None self._ets = TransactionSystem( Path(datadir) / 'transaction_system', EthereumConfig, ) self._ets.backwards_compatibility_tx_storage(Path(datadir)) self.concent_variant = concent_variant self.rpc_router: Optional[CrossbarRouter] = None self.rpc_session: Optional[Session] = None self._rpc_publisher: Optional[Publisher] = None self._peers: List[SocketAddress] = peers or [] # Initialize database self._db = Database(db, fields=DB_FIELDS, models=DB_MODELS, db_dir=datadir) self.client: Optional[Client] = None self.apps_manager = AppsManager() self._client_factory = lambda keys_auth: Client( datadir=datadir, app_config=app_config, config_desc=config_desc, keys_auth=keys_auth, database=self._db, transaction_system=self._ets, use_docker_manager=use_docker_manager, use_monitor=self._use_monitor, concent_variant=concent_variant, geth_address=geth_address, apps_manager=self.apps_manager, task_finished_cb=self._try_shutdown) if password is not None: if not self.set_password(password): raise Exception("Password incorrect") def start(self) -> None: try: rpc = self._start_rpc() def on_rpc_ready() -> Deferred: terms_ = self._check_terms() keys = self._start_keys_auth() docker = self._start_docker() return gatherResults([terms_, keys, docker], consumeErrors=True) chain_function(rpc, on_rpc_ready).addCallbacks( self._setup_client, self._error('keys or docker'), ).addErrback(self._error('setup client')) self._reactor.run() except Exception: # pylint: disable=broad-except logger.exception("Application error") @rpc_utils.expose('ui.quit') def quit(self) -> None: def _quit(): docker_manager = self._docker_manager if docker_manager: docker_manager.quit() reactor = self._reactor if reactor.running: reactor.callFromThread(reactor.stop) # Call in a separate thread and return early from threading import Thread Thread(target=_quit).start() @rpc_utils.expose('golem.password.set') def set_password(self, password: str) -> bool: logger.info("Got password") try: self._keys_auth = KeysAuth( datadir=self._datadir, private_key_name=PRIVATE_KEY, password=password, difficulty=self._config_desc.key_difficulty, ) # When Golem is ready to use different Ethereum account for # payments and identity this should be called only when # idendity was not just created above for the first time. self._ets.backwards_compatibility_privkey( self._keys_auth._private_key, # noqa pylint: disable=protected-access password, ) self._ets.set_password(password) except WrongPassword: logger.info("Password incorrect") return False return True @rpc_utils.expose('golem.password.key_exists') def key_exists(self) -> bool: return KeysAuth.key_exists(self._datadir, PRIVATE_KEY) @rpc_utils.expose('golem.password.unlocked') def is_account_unlocked(self) -> bool: return self._keys_auth is not None @rpc_utils.expose('golem.mainnet') @classmethod def is_mainnet(cls) -> bool: return IS_MAINNET def _start_rpc(self) -> Deferred: self.rpc_router = rpc = CrossbarRouter( host=self._config_desc.rpc_address, port=self._config_desc.rpc_port, datadir=self._datadir, ) self._reactor.addSystemEventTrigger("before", "shutdown", rpc.stop) deferred = rpc.start(self._reactor) return chain_function(deferred, self._start_session) def _start_session(self) -> Optional[Deferred]: if not self.rpc_router: self._stop_on_error("rpc", "RPC router is not available") return None crsb_user = self.rpc_router.cert_manager.CrossbarUsers.golemapp self.rpc_session = Session( self.rpc_router.address, cert_manager=self.rpc_router.cert_manager, use_ipv6=self._config_desc.use_ipv6, crsb_user=crsb_user, crsb_user_secret=self.rpc_router.cert_manager.get_secret( crsb_user)) deferred = self.rpc_session.connect() def on_connect(*_): methods = rpc_utils.object_method_map(self) methods['sys.exposed_procedures'] = \ self.rpc_session.exposed_procedures self.rpc_session.add_procedures(methods) self._rpc_publisher = Publisher(self.rpc_session) StatusPublisher.set_publisher(self._rpc_publisher) return deferred.addCallbacks(on_connect, self._error('rpc session')) @rpc_utils.expose('golem.terms') @staticmethod def are_terms_accepted(): return terms.TermsOfUse.are_accepted() @rpc_utils.expose('golem.concent.terms') @classmethod def are_concent_terms_accepted(cls): return terms.ConcentTermsOfUse.are_accepted() @rpc_utils.expose('golem.terms.accept') def accept_terms(self, enable_monitor: Optional[bool] = None, enable_talkback: Optional[bool] = None) -> None: if enable_talkback is not None: self._config_desc.enable_talkback = enable_talkback self._use_talkback = enable_talkback if enable_monitor is not None: self._config_desc.enable_monitor = enable_monitor self._use_monitor = enable_monitor self._app_config.change_config(self._config_desc) return terms.TermsOfUse.accept() @rpc_utils.expose('golem.concent.terms.accept') @classmethod def accept_concent_terms(cls): return terms.ConcentTermsOfUse.accept() @rpc_utils.expose('golem.terms.show') @staticmethod def show_terms(): return terms.TermsOfUse.show() @rpc_utils.expose('golem.concent.terms.show') @classmethod def show_concent_terms(cls): return terms.ConcentTermsOfUse.show() @rpc_utils.expose('golem.version') @staticmethod def get_golem_version(): return golem.__version__ @rpc_utils.expose('golem.graceful_shutdown') def graceful_shutdown(self) -> ShutdownResponse: if self.client is None: logger.warning('Shutdown called when client=None, try again later') return ShutdownResponse.off # is in shutdown? turn off as toggle if self._config_desc.in_shutdown: self.client.update_setting('in_shutdown', False) logger.info('Turning off shutdown mode') return ShutdownResponse.off # is not providing nor requesting, normal shutdown if not self._is_task_in_progress(): logger.info('Node not working, executing normal shutdown') self.quit() return ShutdownResponse.quit # configure in_shutdown logger.info('Enabling shutdown mode, no more tasks can be started') self.client.update_setting('in_shutdown', True) # subscribe to events return ShutdownResponse.on def _try_shutdown(self) -> None: # is not in shutdown? if not self._config_desc.in_shutdown: logger.debug('Checking shutdown, no shutdown configure') return if self._is_task_in_progress(): logger.info('Shutdown checked, a task is still in progress') return logger.info('Node done with all tasks, shutting down') self.quit() def _is_task_in_progress(self) -> bool: if self.client is None: logger.debug('_is_task_in_progress? False: client=None') return False task_server = self.client.task_server if task_server is None or task_server.task_manager is None: logger.debug('_is_task_in_progress? False: task_manager=None') return False task_requestor_progress = task_server.task_manager.get_progresses() if task_requestor_progress: logger.debug('_is_task_in_progress? requestor=%r', True) return True if task_server.task_computer is None: logger.debug('_is_task_in_progress? False: task_computer=None') return False task_provider_progress = task_server.task_computer.assigned_subtask logger.debug('_is_task_in_progress? provider=%r, requestor=False', task_provider_progress) return bool(task_provider_progress) @require_rpc_session() def _check_terms(self) -> Optional[Deferred]: def wait_for_terms(): sleep_time = 5 while not self.are_terms_accepted() and self._reactor.running: logger.info( 'Terms of use must be accepted before using Golem. ' 'Run `golemcli terms show` to display the terms ' 'and `golemcli terms accept` to accept them.') time.sleep(sleep_time) if None in self.concent_variant.values(): return # Concent disabled while not terms.ConcentTermsOfUse.are_accepted() \ and self._reactor.running: logger.info( 'Concent terms of use must be accepted before using' ' Concent service.' ' Run `golemcli concent terms show`' ' to display the terms' ' and `golemcli concent terms accept` to accept them.', ) time.sleep(sleep_time) return threads.deferToThread(wait_for_terms) @require_rpc_session() def _start_keys_auth(self) -> Optional[Deferred]: def create_keysauth(): # If keys_auth already exists it means we used command line flag # and don't need to inform client about required password if self.is_account_unlocked(): return tip_msg = 'Run `golemcli account unlock` and enter your password.' if self.key_exists(): event = 'get_password' tip_msg = 'Waiting for password to unlock the account. ' \ f'{tip_msg}' else: event = 'new_password' tip_msg = 'New account, waiting for password to be set. ' \ f'{tip_msg}' while not self.is_account_unlocked() and self._reactor.running: logger.info(tip_msg) StatusPublisher.publish(Component.client, event, Stage.pre) time.sleep(5) StatusPublisher.publish(Component.client, event, Stage.post) return threads.deferToThread(create_keysauth) def _start_docker(self) -> Optional[Deferred]: if not self._use_docker_manager: return None def start_docker(): self._docker_manager = DockerManager.install(self._config_desc) self._docker_manager.check_environment() # noqa pylint: disable=no-member return threads.deferToThread(start_docker) @require_rpc_session() def _setup_client(self, *_) -> None: if not self._keys_auth: self._error("KeysAuth is not available") return from golem.tools.talkback import enable_sentry_logger enable_sentry_logger(self._use_talkback) self.client = self._client_factory(self._keys_auth) self._reactor.addSystemEventTrigger("before", "shutdown", self.client.quit) self.client.set_rpc_publisher(self._rpc_publisher) golem_async.async_run( golem_async.AsyncRequest(self._run), error=self._error('Cannot start the client'), ) @require_rpc_session() def _run(self, *_) -> None: if not self.client: self._stop_on_error("client", "Client is not available") return self._setup_apps() self.client.sync() try: self.client.start() for peer in self._peers: self.client.connect(peer) except SystemExit: self._reactor.callFromThread(self._reactor.stop) return methods = self.client.get_wamp_rpc_mapping() def rpc_ready(_): logger.info('All procedures registered in WAMP router') self._rpc_publisher.publish( rpceventnames.Golem.procedures_registered, ) # pylint: disable=no-member self.rpc_session.add_procedures(methods).addCallback( # type: ignore rpc_ready, ) # pylint: enable=no-member def _setup_apps(self) -> None: if not self.client: self._stop_on_error("client", "Client is not available") return self.apps_manager.load_all_apps() for env in self.apps_manager.get_env_list(): env.accept_tasks = True self.client.environments_manager.add_environment(env) def _error(self, msg: str) -> Callable: return functools.partial(self._stop_on_error, msg) def _stop_on_error(self, msg: str, err: Any) -> None: if self._reactor.running: exc_info = (err.type, err.value, err.getTracebackObject()) \ if isinstance(err, Failure) else None logger.error("Stopping because of %r error: %r", msg, err, exc_info=exc_info) self._reactor.callFromThread(self._reactor.stop)
def start_client(start_ranking, datadir=None, transaction_system=False, client=None, reactor=None, **config_overrides): config_logging("client", datadir=datadir) logger = logging.getLogger("golem.client") install_unhandled_error_logger() if not reactor: from twisted.internet import reactor process_monitor = None from golem.core.processmonitor import ProcessMonitor from golem.docker.manager import DockerManager from golem.rpc.router import CrossbarRouter process_monitor = None if not client: client = Client(datadir=datadir, transaction_system=transaction_system, **config_overrides) docker_manager = DockerManager.install(client.config_desc) docker_manager.check_environment() environments = load_environments() client.sync() for env in environments: client.environments_manager.add_environment(env) client.environments_manager.load_config(client.datadir) config = client.config_desc methods = object_method_map(client, CORE_METHOD_MAP) router = CrossbarRouter( host=config.rpc_address, port=config.rpc_port, datadir=client.datadir ) session = Session(router.address, methods=methods) def router_ready(*_): session.connect().addCallbacks(session_ready, start_error) def session_ready(*_): global process_monitor logger.info('Router session ready. Starting client...') try: client.configure_rpc(session) logger.debug('client.start()') client.start() logger.debug('after client.start()') except SystemExit: raise except Exception as exc: logger.exception("Client process error: {}" .format(exc)) logger.info('Starting GUI process...') gui_process = start_gui(router.address) process_monitor = ProcessMonitor(gui_process) process_monitor.add_callbacks(stop_reactor) logger.info('Starting process monitor...') process_monitor.start() reactor.addSystemEventTrigger("before", "shutdown", router.stop) router.start(reactor, router_ready, start_error) if start_ranking: client.ranking.run(reactor) try: reactor.run() except ReactorAlreadyRunning: logger.debug("Client process: reactor is already running") if process_monitor: process_monitor.exit()