class WebSocketCLI(object):
class NoConnection(object):
def __getattribute__(self, item):
raise Exception("Cannot connect to Golem instance")
def __init__(self, cli, host, port, realm, ssl=False):
address = WebSocketAddress(host, port, realm, ssl)
self.cli = cli
self.session = Session(address)
def execute(self, *args, **kwargs):
from twisted.internet import reactor, threads
def on_connected(_):
core_client = Client(self.session, CORE_METHOD_MAP)
self.cli.register_client(core_client)
threads.deferToThread(self.cli.execute, *args, **kwargs).addBoth(self.shutdown)
def on_error(_):
self.cli.register_client(WebSocketCLI.NoConnection())
self.cli.execute(*args, **kwargs)
self.shutdown()
def connect():
self.session.connect().addCallbacks(on_connected, on_error)
reactor.callWhenRunning(connect)
reactor.run()
def shutdown(self, *_):
from twisted.internet import reactor
from twisted.internet.error import ReactorNotRunning
if self.cli:
self.cli.shutdown()
if self.session:
self.session.disconnect()
try:
reactor.stop()
except ReactorNotRunning:
pass
class RPCClient:
def __init__(self,
datadir: str,
host: str = CROSSBAR_HOST,
port: int = CROSSBAR_PORT,
realm: str = CROSSBAR_REALM,
ssl: bool = True) -> None:
address = WebSocketAddress(host, port, realm, ssl)
cert_manager = CertificateManager(
os.path.join(get_local_datadir(datadir), CROSSBAR_DIR))
crsb_user = cert_manager.CrossbarUsers.golemcli
secret = cert_manager.get_secret(crsb_user)
self.session = Session(
address,
crsb_user=crsb_user,
crsb_user_secret=secret,
)
def call(self,
method,
*args,
on_success=lambda: None,
on_error=None,
**kwargs):
def on_connected(_):
def default_errback(error):
print("Error: {}".format(error))
deferred = self.session.call(method, *args, **kwargs)
deferred.addCallbacks(on_success, on_error or default_errback)
deferred.addBoth(self.shutdown)
def connection_error(error):
sys.stderr.write(
"Error connecting to Golem instance ({}): {}\n".format(
self.session.address, error))
self.shutdown()
return self.session.connect(auto_reconnect=False).addCallbacks(
on_connected, on_error or connection_error)
def shutdown(self, *_):
if self.session:
self.session.disconnect()
class Node(object):
""" Simple Golem Node connecting console user interface with Client
:type client golem.client.Client:
"""
def __init__(self,
datadir=None,
transaction_system=False,
**config_overrides):
self.default_environments = []
self.client = Client(datadir=datadir,
transaction_system=transaction_system,
**config_overrides)
self.rpc_router = None
self.rpc_session = None
import logging
self.logger = logging.getLogger("app")
def initialize(self):
self.load_environments(self.default_environments)
self.client.sync()
self.client.start()
def load_environments(self, environments):
for env in environments:
env.accept_tasks = True
self.client.environments_manager.add_environment(env)
def connect_with_peers(self, peers):
for peer in peers:
self.client.connect(peer)
def add_tasks(self, tasks):
for task_def in tasks:
task_builder = self._get_task_builder(task_def)
golem_task = Task.build_task(
task_builder(self.client.get_node_name(), task_def,
self.client.datadir))
self.client.enqueue_new_task(golem_task)
def run(self, use_rpc=False):
try:
from twisted.internet import reactor
if use_rpc:
config = self.client.config_desc
reactor.callWhenRunning(self._start_rpc_server,
config.rpc_address,
int(config.rpc_port))
reactor.run()
except Exception as ex:
self.logger.error("Reactor error: {}".format(ex))
finally:
self.client.quit()
sys.exit(0)
def _start_rpc_server(self, host, port):
from twisted.internet import reactor
from golem.rpc.router import CrossbarRouter
self.rpc_router = CrossbarRouter(host=host,
port=port,
datadir=self.client.datadir)
reactor.addSystemEventTrigger("before", "shutdown",
self.rpc_router.stop)
self.rpc_router.start(reactor, self._router_ready, self._rpc_error)
def _router_ready(self, *_):
methods = object_method_map(self.client, CORE_METHOD_MAP)
self.rpc_session = Session(self.rpc_router.address, methods=methods)
self.client.configure_rpc(self.rpc_session)
self.rpc_session.connect().addErrback(self._rpc_error)
def _rpc_error(self, err):
self.logger.error("RPC error: {}".format(err))
def _get_task_builder(self, task_def):
raise NotImplementedError
class WebSocketCLI(object):
class NoConnection(object):
def __getattribute__(self, item):
raise Exception("Cannot connect to Golem instance")
class CLIClient(ClientProxy):
def _call(self, method_alias, *args, **kwargs):
from twisted.internet import reactor
deferred = Deferred()
super_call = super()._call
def in_reactor():
try:
parent_deferred = super_call(
method_alias,
*args,
**kwargs,
)
except Exception as exc: # pylint: disable=broad-except
deferred.errback(exc)
else:
parent_deferred.chainDeferred(deferred)
reactor.callFromThread(in_reactor)
return deferred
def __init__(
self,
cli, # pylint: disable=too-many-arguments
cert_manager: CertificateManager,
host: str = CROSSBAR_HOST,
port: int = CROSSBAR_PORT,
realm: str = CROSSBAR_REALM,
ssl: bool = True) -> None:
address = WebSocketAddress(host, port, realm, ssl)
self.cli = cli
crsb_user = cert_manager.CrossbarUsers.golemcli
self.session = Session(
address,
crsb_user=crsb_user,
crsb_user_secret=cert_manager.get_secret(crsb_user))
def execute(self, *args, **kwargs):
from twisted.internet import reactor, threads
def on_ready(_):
threads.deferToThread(self.cli.execute, *args, **kwargs) \
.addBoth(self.shutdown)
def on_connected(_):
core_client = WebSocketCLI.CLIClient(self.session)
self.cli.register_client(core_client)
core_client._ready.addCallback(on_ready) # noqa pylint: disable=protected-access
def on_error(error):
sys.stderr.write(
"Error connecting to Golem instance ({}): {}\n".format(
self.session.address, error))
self.cli.register_client(WebSocketCLI.NoConnection())
self.cli.execute(*args, **kwargs)
self.shutdown()
def connect():
self.session.connect(auto_reconnect=False).addCallbacks(
on_connected, on_error)
reactor.callWhenRunning(connect)
reactor.run()
def shutdown(self, *_):
from twisted.internet import reactor
from twisted.internet.error import ReactorNotRunning
if self.cli:
self.cli.shutdown()
if self.session:
self.session.disconnect()
try:
reactor.stop()
except ReactorNotRunning:
pass
class Node(object):
""" Simple Golem Node connecting console user interface with Client
:type client golem.client.Client:
"""
def __init__(
self, # noqa pylint: disable=too-many-arguments
datadir: str,
app_config: AppConfig,
config_desc: ClientConfigDescriptor,
# SEE golem.core.variables.CONCENT_CHOICES
concent_variant: dict,
peers: Optional[List[SocketAddress]] = None,
use_monitor: bool = None,
use_talkback: bool = None,
use_docker_manager: bool = True,
geth_address: Optional[str] = None,
password: Optional[str] = None) -> None:
# DO NOT MAKE THIS IMPORT GLOBAL
# otherwise, reactor will install global signal handlers on import
# and will prevent the IOCP / kqueue reactors from being installed.
from twisted.internet import reactor
self._reactor = reactor
self._app_config = app_config
self._config_desc = config_desc
self._datadir = datadir
self._use_docker_manager = use_docker_manager
self._docker_manager: Optional[DockerManager] = None
self._use_monitor = config_desc.enable_monitor \
if use_monitor is None else use_monitor
self._use_talkback = config_desc.enable_talkback \
if use_talkback is None else use_talkback
self._keys_auth: Optional[KeysAuth] = None
self._ets = TransactionSystem(
Path(datadir) / 'transaction_system',
EthereumConfig,
)
self._ets.backwards_compatibility_tx_storage(Path(datadir))
self.concent_variant = concent_variant
self.rpc_router: Optional[CrossbarRouter] = None
self.rpc_session: Optional[Session] = None
self._rpc_publisher: Optional[Publisher] = None
self._peers: List[SocketAddress] = peers or []
# Initialize database
self._db = Database(db,
fields=DB_FIELDS,
models=DB_MODELS,
db_dir=datadir)
self.client: Optional[Client] = None
self.apps_manager = AppsManager()
self._client_factory = lambda keys_auth: Client(
datadir=datadir,
app_config=app_config,
config_desc=config_desc,
keys_auth=keys_auth,
database=self._db,
transaction_system=self._ets,
use_docker_manager=use_docker_manager,
use_monitor=self._use_monitor,
concent_variant=concent_variant,
geth_address=geth_address,
apps_manager=self.apps_manager,
task_finished_cb=self._try_shutdown)
if password is not None:
if not self.set_password(password):
raise Exception("Password incorrect")
def start(self) -> None:
try:
rpc = self._start_rpc()
def on_rpc_ready() -> Deferred:
terms_ = self._check_terms()
keys = self._start_keys_auth()
docker = self._start_docker()
return gatherResults([terms_, keys, docker],
consumeErrors=True)
chain_function(rpc, on_rpc_ready).addCallbacks(
self._setup_client,
self._error('keys or docker'),
).addErrback(self._error('setup client'))
self._reactor.run()
except Exception: # pylint: disable=broad-except
logger.exception("Application error")
@rpc_utils.expose('ui.quit')
def quit(self) -> None:
def _quit():
docker_manager = self._docker_manager
if docker_manager:
docker_manager.quit()
reactor = self._reactor
if reactor.running:
reactor.callFromThread(reactor.stop)
# Call in a separate thread and return early
from threading import Thread
Thread(target=_quit).start()
@rpc_utils.expose('golem.password.set')
def set_password(self, password: str) -> bool:
logger.info("Got password")
try:
self._keys_auth = KeysAuth(
datadir=self._datadir,
private_key_name=PRIVATE_KEY,
password=password,
difficulty=self._config_desc.key_difficulty,
)
# When Golem is ready to use different Ethereum account for
# payments and identity this should be called only when
# idendity was not just created above for the first time.
self._ets.backwards_compatibility_privkey(
self._keys_auth._private_key, # noqa pylint: disable=protected-access
password,
)
self._ets.set_password(password)
except WrongPassword:
logger.info("Password incorrect")
return False
return True
@rpc_utils.expose('golem.password.key_exists')
def key_exists(self) -> bool:
return KeysAuth.key_exists(self._datadir, PRIVATE_KEY)
@rpc_utils.expose('golem.password.unlocked')
def is_account_unlocked(self) -> bool:
return self._keys_auth is not None
@rpc_utils.expose('golem.mainnet')
@classmethod
def is_mainnet(cls) -> bool:
return IS_MAINNET
def _start_rpc(self) -> Deferred:
self.rpc_router = rpc = CrossbarRouter(
host=self._config_desc.rpc_address,
port=self._config_desc.rpc_port,
datadir=self._datadir,
)
self._reactor.addSystemEventTrigger("before", "shutdown", rpc.stop)
deferred = rpc.start(self._reactor)
return chain_function(deferred, self._start_session)
def _start_session(self) -> Optional[Deferred]:
if not self.rpc_router:
self._stop_on_error("rpc", "RPC router is not available")
return None
crsb_user = self.rpc_router.cert_manager.CrossbarUsers.golemapp
self.rpc_session = Session(
self.rpc_router.address,
cert_manager=self.rpc_router.cert_manager,
use_ipv6=self._config_desc.use_ipv6,
crsb_user=crsb_user,
crsb_user_secret=self.rpc_router.cert_manager.get_secret(
crsb_user))
deferred = self.rpc_session.connect()
def on_connect(*_):
methods = rpc_utils.object_method_map(self)
methods['sys.exposed_procedures'] = \
self.rpc_session.exposed_procedures
self.rpc_session.add_procedures(methods)
self._rpc_publisher = Publisher(self.rpc_session)
StatusPublisher.set_publisher(self._rpc_publisher)
return deferred.addCallbacks(on_connect, self._error('rpc session'))
@rpc_utils.expose('golem.terms')
@staticmethod
def are_terms_accepted():
return terms.TermsOfUse.are_accepted()
@rpc_utils.expose('golem.concent.terms')
@classmethod
def are_concent_terms_accepted(cls):
return terms.ConcentTermsOfUse.are_accepted()
@rpc_utils.expose('golem.terms.accept')
def accept_terms(self,
enable_monitor: Optional[bool] = None,
enable_talkback: Optional[bool] = None) -> None:
if enable_talkback is not None:
self._config_desc.enable_talkback = enable_talkback
self._use_talkback = enable_talkback
if enable_monitor is not None:
self._config_desc.enable_monitor = enable_monitor
self._use_monitor = enable_monitor
self._app_config.change_config(self._config_desc)
return terms.TermsOfUse.accept()
@rpc_utils.expose('golem.concent.terms.accept')
@classmethod
def accept_concent_terms(cls):
return terms.ConcentTermsOfUse.accept()
@rpc_utils.expose('golem.terms.show')
@staticmethod
def show_terms():
return terms.TermsOfUse.show()
@rpc_utils.expose('golem.concent.terms.show')
@classmethod
def show_concent_terms(cls):
return terms.ConcentTermsOfUse.show()
@rpc_utils.expose('golem.version')
@staticmethod
def get_golem_version():
return golem.__version__
@rpc_utils.expose('golem.graceful_shutdown')
def graceful_shutdown(self) -> ShutdownResponse:
if self.client is None:
logger.warning('Shutdown called when client=None, try again later')
return ShutdownResponse.off
# is in shutdown? turn off as toggle
if self._config_desc.in_shutdown:
self.client.update_setting('in_shutdown', False)
logger.info('Turning off shutdown mode')
return ShutdownResponse.off
# is not providing nor requesting, normal shutdown
if not self._is_task_in_progress():
logger.info('Node not working, executing normal shutdown')
self.quit()
return ShutdownResponse.quit
# configure in_shutdown
logger.info('Enabling shutdown mode, no more tasks can be started')
self.client.update_setting('in_shutdown', True)
# subscribe to events
return ShutdownResponse.on
def _try_shutdown(self) -> None:
# is not in shutdown?
if not self._config_desc.in_shutdown:
logger.debug('Checking shutdown, no shutdown configure')
return
if self._is_task_in_progress():
logger.info('Shutdown checked, a task is still in progress')
return
logger.info('Node done with all tasks, shutting down')
self.quit()
def _is_task_in_progress(self) -> bool:
if self.client is None:
logger.debug('_is_task_in_progress? False: client=None')
return False
task_server = self.client.task_server
if task_server is None or task_server.task_manager is None:
logger.debug('_is_task_in_progress? False: task_manager=None')
return False
task_requestor_progress = task_server.task_manager.get_progresses()
if task_requestor_progress:
logger.debug('_is_task_in_progress? requestor=%r', True)
return True
if task_server.task_computer is None:
logger.debug('_is_task_in_progress? False: task_computer=None')
return False
task_provider_progress = task_server.task_computer.assigned_subtask
logger.debug('_is_task_in_progress? provider=%r, requestor=False',
task_provider_progress)
return bool(task_provider_progress)
@require_rpc_session()
def _check_terms(self) -> Optional[Deferred]:
def wait_for_terms():
sleep_time = 5
while not self.are_terms_accepted() and self._reactor.running:
logger.info(
'Terms of use must be accepted before using Golem. '
'Run `golemcli terms show` to display the terms '
'and `golemcli terms accept` to accept them.')
time.sleep(sleep_time)
if None in self.concent_variant.values():
return # Concent disabled
while not terms.ConcentTermsOfUse.are_accepted() \
and self._reactor.running:
logger.info(
'Concent terms of use must be accepted before using'
' Concent service.'
' Run `golemcli concent terms show`'
' to display the terms'
' and `golemcli concent terms accept` to accept them.', )
time.sleep(sleep_time)
return threads.deferToThread(wait_for_terms)
@require_rpc_session()
def _start_keys_auth(self) -> Optional[Deferred]:
def create_keysauth():
# If keys_auth already exists it means we used command line flag
# and don't need to inform client about required password
if self.is_account_unlocked():
return
tip_msg = 'Run `golemcli account unlock` and enter your password.'
if self.key_exists():
event = 'get_password'
tip_msg = 'Waiting for password to unlock the account. ' \
f'{tip_msg}'
else:
event = 'new_password'
tip_msg = 'New account, waiting for password to be set. ' \
f'{tip_msg}'
while not self.is_account_unlocked() and self._reactor.running:
logger.info(tip_msg)
StatusPublisher.publish(Component.client, event, Stage.pre)
time.sleep(5)
StatusPublisher.publish(Component.client, event, Stage.post)
return threads.deferToThread(create_keysauth)
def _start_docker(self) -> Optional[Deferred]:
if not self._use_docker_manager:
return None
def start_docker():
self._docker_manager = DockerManager.install(self._config_desc)
self._docker_manager.check_environment() # noqa pylint: disable=no-member
return threads.deferToThread(start_docker)
@require_rpc_session()
def _setup_client(self, *_) -> None:
if not self._keys_auth:
self._error("KeysAuth is not available")
return
from golem.tools.talkback import enable_sentry_logger
enable_sentry_logger(self._use_talkback)
self.client = self._client_factory(self._keys_auth)
self._reactor.addSystemEventTrigger("before", "shutdown",
self.client.quit)
self.client.set_rpc_publisher(self._rpc_publisher)
golem_async.async_run(
golem_async.AsyncRequest(self._run),
error=self._error('Cannot start the client'),
)
@require_rpc_session()
def _run(self, *_) -> None:
if not self.client:
self._stop_on_error("client", "Client is not available")
return
self._setup_apps()
self.client.sync()
try:
self.client.start()
for peer in self._peers:
self.client.connect(peer)
except SystemExit:
self._reactor.callFromThread(self._reactor.stop)
return
methods = self.client.get_wamp_rpc_mapping()
def rpc_ready(_):
logger.info('All procedures registered in WAMP router')
self._rpc_publisher.publish(
rpceventnames.Golem.procedures_registered, )
# pylint: disable=no-member
self.rpc_session.add_procedures(methods).addCallback( # type: ignore
rpc_ready, )
# pylint: enable=no-member
def _setup_apps(self) -> None:
if not self.client:
self._stop_on_error("client", "Client is not available")
return
self.apps_manager.load_all_apps()
for env in self.apps_manager.get_env_list():
env.accept_tasks = True
self.client.environments_manager.add_environment(env)
def _error(self, msg: str) -> Callable:
return functools.partial(self._stop_on_error, msg)
def _stop_on_error(self, msg: str, err: Any) -> None:
if self._reactor.running:
exc_info = (err.type, err.value, err.getTracebackObject()) \
if isinstance(err, Failure) else None
logger.error("Stopping because of %r error: %r",
msg,
err,
exc_info=exc_info)
self._reactor.callFromThread(self._reactor.stop)