def test_comparison(self):
k = EllipticalKeysAuth(self.path)
e = urandom(20)
a = EthAccountInfo(k.get_key_id(), 5111, "10.0.0.1", "test-test-test", Node(), e)
b = EthAccountInfo(k.get_key_id(), 5111, "10.0.0.1", "test-test-test", Node(), e)
self.assertEqual(a, b)
n = Node(prv_addr="10.10.10.10", prv_port=1031, pub_addr="10.10.10.10", pub_port=1032)
c = EthAccountInfo(k.get_key_id(), 5112, "10.0.0.2", "test-test2-test", n, e)
self.assertEqual(a, c)
k.generate_new(2)
c.key_id = k.get_key_id()
self.assertNotEqual(a, c)
def test_verify(self):
conn = MagicMock()
peer_session = PeerSession(conn)
keys_auth = EllipticalKeysAuth(self.path)
peer_session.key_id = keys_auth.get_key_id()
peer_session.p2p_service.verify_sig = keys_auth.verify
msg = MessageStopGossip()
assert not peer_session.verify(msg)
msg.sig = keys_auth.sign(msg.get_short_hash())
assert peer_session.verify(msg)
def test_verify(self):
keys_auth = EllipticalKeysAuth(self.path)
conn = Mock()
ts = TaskSession(conn)
ts.task_server = Mock()
ts.task_server.verify_sig = keys_auth.verify
msg = message.MessageRemoveTask()
assert not ts.verify(msg)
msg.sig = keys_auth.sign(msg.get_short_hash())
ts.key_id = keys_auth.get_key_id()
assert ts.verify(msg)
class TestP2PService(testutils.DatabaseFixture, testutils.PEP8MixIn):
PEP8_FILES = ['golem/network/p2p/p2pservice.py',]
def setUp(self):
super(TestP2PService, self).setUp()
random.seed()
self.keys_auth = EllipticalKeysAuth(self.path)
self.service = P2PService(None, ClientConfigDescriptor(), self.keys_auth, connect_to_known_hosts=False)
def test_find_node(self):
node_key_id = uuid.uuid4()
# find_node() without parameter
node_session = peersession.PeerSession(conn=mock.MagicMock())
node_session.listen_port = random.randint(1, 2**16-1)
node_session.address = random.randint(1, 2**32-1)
node_session.node_name = 'approximately 16.8 million addresses'
node_session.node_info = None
self.service.peers = {
node_key_id: peersession.PeerSessionInfo(node_session),
}
expected = [
{
'address': node_session.address,
'port': node_session.listen_port,
'node_name': node_session.node_name,
'node': None,
},
]
self.assertEquals(self.service.find_node(node_key_id=None), expected)
# find_node() via kademlia neighbours
neighbour_node_key_id = uuid.uuid4()
neighbour_node = Node(
node_name=u'Syndrom wstrząsu toksycznego',
key=neighbour_node_key_id,
prv_addr=random.randint(1, 2**32-1),
prv_port=random.randint(1, 2**16-1)
)
self.service.peer_keeper.neighbours = mock.MagicMock(return_value=[
neighbour_node,
])
expected = [
{
'address': neighbour_node.prv_addr,
'port': neighbour_node.prv_port,
'id': neighbour_node.key,
'node': neighbour_node,
'node_name': neighbour_node.node_name,
},
]
self.assertEquals(self.service.find_node(node_key_id), expected)
def test_add_to_peer_keeper(self):
node = Node()
node.key = EllipticalKeysAuth("TEST").get_key_id()
m_test2 = MagicMock()
m_test3 = MagicMock()
self.service.peers["TEST3"] = m_test3
self.service.peers["TEST2"] = m_test2
self.service.peer_keeper = MagicMock()
node2 = Node()
node2.key = "TEST2"
self.service.peer_keeper.add_peer = MagicMock(return_value=node2)
self.service.add_to_peer_keeper(node)
m_test2.ping.assert_called_with(0)
m_test3.ping.assert_not_called()
for i in range(100):
self.service.peers[str(i)] = MagicMock()
node2.key = "59"
self.service.add_to_peer_keeper(node)
self.service.peers["59"].ping.assert_called_with(0)
for i in range(58) + range(60, 100):
self.service.peers[str(i)].ping.assert_not_called()
node2.key = None
self.service.add_to_peer_keeper(node)
for i in range(58) + range(60, 100):
self.service.peers[str(i)].ping.assert_not_called()
self.service.peers["59"].ping.assert_called_once_with(0)
m_test2.ping.assert_called_once_with(0)
m_test3.ping.assert_not_called()
self.assertEqual(len(self.service.peers), 102)
def test_remove_old_peers(self):
node = MagicMock()
node.key = EllipticalKeysAuth(self.path, "TESTPRIV", "TESTPUB").get_key_id()
node.key_id = node.key
self.service.last_peers_request = time.time() + 10
self.service.add_peer(node.key, node)
assert len(self.service.peers) == 1
node.last_message_time = 0
self.service.sync_network()
assert len(self.service.peers) == 0
self.service.add_peer(node.key, node)
self.service.peers[node.key].last_message_time = time.time() + 1000
assert len(self.service.peers) == 1
self.service.sync_network()
assert len(self.service.peers) == 1
def test_refresh_peers(self):
sa = SocketAddress('127.0.0.1', 11111)
node = MagicMock()
node.key = EllipticalKeysAuth(self.path, "TESTPRIV", "TESTPUB").get_key_id()
node.key_id = node.key
node.address = sa
node2 = MagicMock()
node2.key = EllipticalKeysAuth(self.path, "TESTPRIV2", "TESTPUB2").get_key_id()
node2.key_id = node2.key
node2.address = sa
self.service.add_peer(node.key, node)
self.service.add_peer(node2.key, node2)
self.service.peers[node.key].last_message_time = time.time() + 1000
self.service.peers[node2.key].last_message_time = time.time() + 1000
self.service.config_desc.opt_peer_num = 1000
assert len(self.service.peers) == 2
self.service.sync_network()
assert len(self.service.peers) == 2
self.service.last_refresh_peers = 0
self.service.last_peers_request = 0
self.service._peer_dbg_time_threshold = 0
self.service.sync_network()
# disabled
assert len(self.service.peers) == 2
def test_add_known_peer(self):
key_id = EllipticalKeysAuth(self.path, "TESTPRIV", "TESTPUB").get_key_id()
nominal_seeds = len(self.service.seeds)
node = Node(
'super_node', key_id,
pub_addr='1.2.3.4',
prv_addr='1.2.3.4',
pub_port=10000,
prv_port=10000
)
node.prv_addresses = [node.prv_addr, '172.1.2.3']
assert Node.is_super_node(node)
KnownHosts.delete().execute()
len_start = len(KnownHosts.select())
# insert one
self.service.add_known_peer(node, node.pub_addr, node.pub_port)
select_1 = KnownHosts.select()
len_1 = len(select_1)
last_conn_1 = select_1[0].last_connected
assert len_1 > len_start
# advance time
time.sleep(0.1)
# insert duplicate
self.service.add_known_peer(node, node.pub_addr, node.pub_port)
select_2 = KnownHosts.select()
len_2 = len(select_2)
assert len_2 == len_1
assert select_2[0].last_connected > last_conn_1
assert len(self.service.seeds) > nominal_seeds
# try to add more than max, we already have at least 1
pub_prefix = '2.2.3.'
prv_prefix = '172.1.2.'
for i in xrange(1, MAX_STORED_HOSTS + 6):
i_str = str(i)
pub = pub_prefix + i_str
prv = prv_prefix + i_str
n = Node(
i_str, key_id + i_str,
pub_addr=pub,
prv_addr=prv,
pub_port=10000,
prv_port=10000
)
self.service.add_known_peer(n, pub, n.prv_port)
assert len(KnownHosts.select()) == MAX_STORED_HOSTS
assert len(self.service.seeds) == nominal_seeds
def test_sync_free_peers(self):
node = MagicMock()
node.key = EllipticalKeysAuth(self.path, "PRIVTEST", "PUBTEST").get_key_id()
node.key_id = node.key
node.pub_addr = '127.0.0.1'
node.pub_port = 10000
self.service.config_desc.opt_peer_num = 10
self.service.free_peers.append(node.key)
self.service.incoming_peers[node.key] = {
'address': '127.0.0.1',
'port': 10000,
'node': node,
'node_name': 'TEST',
'conn_trials': 0
}
self.service.last_peers_request = time.time() - 60
self.service.sync_network()
assert not self.service.free_peers
assert len(self.service.pending_connections) == 1
def test_reconnect_with_seed(self):
self.service.connect_to_seeds()
time_ = time.time()
last_time = self.service.last_time_tried_connect_with_seed
assert self.service.last_time_tried_connect_with_seed <= time_
assert time_ - self.service.last_time_tried_connect_with_seed < self.service.reconnect_with_seed_threshold
assert len(self.service.peers) == 0
self.service.sync_network()
assert last_time == self.service.last_time_tried_connect_with_seed
self.service.reconnect_with_seed_threshold = 0.1
time.sleep(0.5)
self.service.sync_network()
assert last_time < self.service.last_time_tried_connect_with_seed
def test_want_to_start_task_session(self):
self.service.task_server = MagicMock()
self.service.task_server.task_connections_helper = TaskConnectionsHelper()
self.service.task_server.task_connections_helper.task_server = self.service.task_server
self.service.task_server.task_connections_helper.is_new_conn_request = Mock(side_effect=lambda *_: True)
def true_method(*args):
return True
key_id = str(uuid.uuid4())
conn_id = str(uuid.uuid4())
peer_id = str(uuid.uuid4())
node_info = MagicMock()
node_info.key = key_id
node_info.is_super_node = true_method
peer = MagicMock()
peer.key_id = str(uuid.uuid4())
self.service.peers[peer_id] = peer
self.service.node = node_info
self.service.want_to_start_task_session(key_id, node_info, conn_id)
assert not peer.send_want_to_start_task_session.called
self.service.want_to_start_task_session(peer.key_id, node_info, conn_id)
assert not peer.send_want_to_start_task_session.called
peer.key_id = peer_id
self.service.want_to_start_task_session(peer.key_id, node_info, conn_id)
assert peer.send_want_to_start_task_session.called
def test_get_diagnostic(self):
m = MagicMock()
m.transport.getPeer.return_value.port = "10432"
m.transport.getPeer.return_value.host = "10.10.10.10"
ps1 = PeerSession(m)
ps1.key_id = self.keys_auth.key_id
self.service.add_peer(self.keys_auth.key_id, ps1)
m2 = MagicMock()
m2.transport.getPeer.return_value.port = "11432"
m2.transport.getPeer.return_value.host = "127.0.0.1"
ps2 = PeerSession(m2)
keys_auth2 = EllipticalKeysAuth(self.path, "PUBTESTPATH1", "PUBTESTPATH2")
ps2.key_id = keys_auth2.key_id
self.service.add_peer(keys_auth2.key_id, ps2)
self.service.get_diagnostics(DiagnosticsOutputFormat.json)
def test(self):
self.service.task_server = Mock()
self.service.peer_keeper = Mock()
self.service.peer_keeper.sync.return_value = dict()
self.service.connect = Mock()
self.service.last_tasks_request = 0
p = Mock()
p.key_id = 'deadbeef'
p.degree = 1
p.last_message_time = 0
p2 = Mock()
p2.key_id = 'deadbeef02'
p2.degree = 1
p2.last_message_time = 0
self.service.peers[p.key_id] = p
self.service.peers['deadbeef02'] = p2
self.service.resource_peers['deadbeef02'] = [1, 2, 3, 4]
self.service.peer_order = [p.key_id, p2.key_id]
self.service.peer_keeper.sessions_to_end = [p2]
self.service.ping_peers(1)
assert p.ping.called
self.service.key_changed()
assert p.dropped.called
degrees = self.service.get_peers_degree()
assert len(degrees) == 2
assert p.key_id in degrees
self.service.send_get_resource_peers()
assert p.send_get_resource_peers.called
resource_peers = self.service.get_resource_peers()
assert len(resource_peers) == 1
self.service.remove_task('task_id')
assert p.send_remove_task.called
self.service.inform_about_nat_traverse_failure(str(uuid.uuid4()), 'res_key_id', 'conn_id')
assert not p.send_inform_about_nat_traverse_failure.called
self.service.inform_about_nat_traverse_failure(p.key_id, 'res_key_id', 'conn_id')
assert p.send_inform_about_nat_traverse_failure.called
self.service.inform_about_task_nat_hole(str(uuid.uuid4()), 'rv_key_id', '127.0.0.1', 40102, 'ans_conn_id')
assert not p.send_task_nat_hole.called
self.service.inform_about_task_nat_hole(p.key_id, 'rv_key_id', '127.0.0.1', 40102, 'ans_conn_id')
assert p.send_task_nat_hole.called
self.service.send_nat_traverse_failure(p.key_id, 'conn_id')
assert p.send_nat_traverse_failure.called
self.service.send_stop_gossip()
assert p.send_stop_gossip.called
self.service.sync_network()
assert p.send_get_tasks.called
self.service.remove_peer(p)
assert p.key_id not in self.service.peers
def test_challenge_history_len(self):
difficulty = self.service._get_difficulty("KEY_ID")
for i in range(3):
challenge = self.service._get_challenge(self.keys_auth.get_key_id())
self.service.solve_challenge(self.keys_auth.get_key_id(), challenge, difficulty)
assert len(self.service.challenge_history) == 3
assert self.service.last_challenge is not None
for i in range(100):
challenge = self.service._get_challenge(self.keys_auth.get_key_id())
self.service.solve_challenge(self.keys_auth.get_key_id(), challenge, difficulty)
assert len(self.service.challenge_history) == HISTORY_LEN
def test_change_config_name(self):
ccd = ClientConfigDescriptor()
ccd.node_name = "test name change"
assert self.service.node_name != "test name change"
self.service.change_config(ccd)
assert self.service.node_name == "test name change"
def test_broadcast_on_name_change(self):
conn = MagicMock()
peer = PeerSession(conn)
peer.hello_called = False
def fake_hello(self):
self.hello_called = True
import types
peer.hello = types.MethodType(fake_hello, peer)
keys_auth = EllipticalKeysAuth(self.path, "PUBTESTPATH1", "PUBTESTPATH2")
peer.key_id = keys_auth.key_id
self.service.add_peer(keys_auth.key_id, peer)
ccd = ClientConfigDescriptor()
assert not peer.hello_called
self.service.change_config(ccd)
assert not peer.hello_called # negative test
ccd = ClientConfigDescriptor()
ccd.node_name = "test sending hello on name change"
self.service.change_config(ccd)
assert peer.hello_called # positive test
class Client(HardwarePresetsMixin):
def __init__(self,
datadir=None,
transaction_system=False,
connect_to_known_hosts=True,
use_docker_machine_manager=True,
use_monitor=True,
**config_overrides):
if not datadir:
datadir = get_local_datadir('default')
self.datadir = datadir
self.__lock_datadir()
self.lock = Lock()
self.task_tester = None
# Read and validate configuration
config = AppConfig.load_config(datadir)
self.config_desc = ClientConfigDescriptor()
self.config_desc.init_from_app_config(config)
for key, val in config_overrides.iteritems():
if not hasattr(self.config_desc, key):
self.quit() # quit only closes underlying services (for now)
raise AttributeError(
"Can't override nonexistent config entry '{}'".format(key))
setattr(self.config_desc, key, val)
self.config_approver = ConfigApprover(self.config_desc)
log.info('Client "%s", datadir: %s', self.config_desc.node_name,
datadir)
# Initialize database
self.db = Database(datadir)
# Hardware configuration
HardwarePresets.initialize(self.datadir)
HardwarePresets.update_config(self.config_desc.hardware_preset_name,
self.config_desc)
self.keys_auth = EllipticalKeysAuth(self.datadir)
# NETWORK
self.node = Node(node_name=self.config_desc.node_name,
prv_addr=self.config_desc.node_address,
key=self.keys_auth.get_key_id())
self.p2pservice = None
self.diag_service = None
self.task_server = None
self.last_nss_time = time.time()
self.last_net_check_time = time.time()
self.last_balance_time = time.time()
self.last_tasks_time = time.time()
self.last_node_state_snapshot = None
self.nodes_manager_client = None
self.do_work_task = task.LoopingCall(self.__do_work)
self.publish_task = task.LoopingCall(self.__publish_events)
self.cfg = config
self.send_snapshot = False
self.snapshot_lock = Lock()
self.ranking = Ranking(self)
if transaction_system:
# Bootstrap transaction system if enabled.
# TODO: Transaction system (and possible other modules) should be
# modeled as a Service that run independently.
# The Client/Application should be a collection of services.
self.transaction_system = EthereumTransactionSystem(
datadir, self.keys_auth._private_key)
else:
self.transaction_system = None
self.use_docker_machine_manager = use_docker_machine_manager
self.connect_to_known_hosts = connect_to_known_hosts
self.environments_manager = EnvironmentsManager()
self.daemon_manager = None
self.rpc_publisher = None
self.ipfs_manager = None
self.resource_server = None
self.resource_port = 0
self.last_get_resource_peers_time = time.time()
self.get_resource_peers_interval = 5.0
self.use_monitor = use_monitor
self.monitor = None
self.session_id = uuid.uuid4().get_hex()
dispatcher.connect(self.p2p_listener, signal='golem.p2p')
dispatcher.connect(self.taskmanager_listener,
signal='golem.taskmanager')
atexit.register(self.quit)
def configure_rpc(self, rpc_session):
self.rpc_publisher = Publisher(rpc_session)
def p2p_listener(self, sender, signal, event='default', **kwargs):
if event != 'unreachable':
return
self.node.port_status = kwargs.get('description', u'')
def taskmanager_listener(self, sender, signal, event='default', **kwargs):
if event != 'task_status_updated':
return
self._publish(Task.evt_task_status, kwargs['task_id'])
# TODO: re-enable
def sync(self):
pass
def start(self):
if self.use_monitor:
self.init_monitor()
try:
self.start_network()
except SystemExit:
raise
except Exception:
log.critical('Can\'t start network. Giving up.', exc_info=True)
sys.exit(1)
self.do_work_task.start(1, False)
self.publish_task.start(1, True)
def start_network(self):
log.info("Starting network ...")
self.node.collect_network_info(self.config_desc.seed_host,
use_ipv6=self.config_desc.use_ipv6)
log.debug("Is super node? %s", self.node.is_super_node())
# self.ipfs_manager = IPFSDaemonManager(
# connect_to_bootstrap_nodes=self.connect_to_known_hosts)
# self.ipfs_manager.store_client_info()
self.p2pservice = P2PService(
self.node,
self.config_desc,
self.keys_auth,
connect_to_known_hosts=self.connect_to_known_hosts)
self.task_server = TaskServer(
self.node,
self.config_desc,
self.keys_auth,
self,
use_ipv6=self.config_desc.use_ipv6,
use_docker_machine_manager=self.use_docker_machine_manager)
dir_manager = self.task_server.task_computer.dir_manager
log.info("Starting resource server ...")
self.daemon_manager = HyperdriveDaemonManager(self.datadir)
hyperdrive_ports = self.daemon_manager.start()
resource_manager = HyperdriveResourceManager(dir_manager)
self.resource_server = BaseResourceServer(resource_manager,
dir_manager, self.keys_auth,
self)
def connect((p2p_port, task_port)):
log.info('P2P server is listening on port %s', p2p_port)
log.info('Task server is listening on port %s', task_port)
dispatcher.send(signal='golem.p2p',
event='listening',
port=[p2p_port, task_port] +
list(hyperdrive_ports))
listener = ClientTaskComputerEventListener(self)
self.task_server.task_computer.register_listener(listener)
self.p2pservice.connect_to_network()
if self.monitor:
self.diag_service.register(self.p2pservice,
self.monitor.on_peer_snapshot)
self.monitor.on_login()
def terminate(*exceptions):
log.error("Golem cannot listen on ports: %s", exceptions)
self.quit()
task = Deferred()
p2p = Deferred()
gatherResults([p2p, task],
consumeErrors=True).addCallbacks(connect, terminate)
log.info("Starting p2p server ...")
self.p2pservice.task_server = self.task_server
self.p2pservice.set_resource_server(self.resource_server)
self.p2pservice.set_metadata_manager(self)
self.p2pservice.start_accepting(listening_established=p2p.callback,
listening_failure=p2p.errback)
log.info("Starting task server ...")
self.task_server.start_accepting(listening_established=task.callback,
listening_failure=task.errback)
def init_monitor(self):
metadata = self.__get_nodemetadatamodel()
self.monitor = SystemMonitor(metadata, MONITOR_CONFIG)
self.monitor.start()
self.diag_service = DiagnosticsService(DiagnosticsOutputFormat.data)
self.diag_service.register(VMDiagnosticsProvider(),
self.monitor.on_vm_snapshot)
self.diag_service.start_looping_call()
def connect(self, socket_address):
if isinstance(socket_address, Iterable):
socket_address = SocketAddress(socket_address[0],
int(socket_address[1]))
log.debug("P2pservice connecting to %s on port %s",
socket_address.address, socket_address.port)
self.p2pservice.connect(socket_address)
def quit(self):
if self.do_work_task.running:
self.do_work_task.stop()
if self.publish_task.running:
self.publish_task.stop()
if self.task_server:
self.task_server.quit()
if self.transaction_system:
self.transaction_system.stop()
if self.diag_service:
self.diag_service.unregister_all()
if self.daemon_manager:
self.daemon_manager.stop()
dispatcher.send(signal='golem.monitor', event='shutdown')
if self.db:
self.db.close()
self._unlock_datadir()
def key_changed(self):
self.node.key = self.keys_auth.get_key_id()
self.task_server.key_changed()
self.p2pservice.key_changed()
def stop_network(self):
# FIXME: Implement this method properly - send disconnect package,
# close connections etc.
self.p2pservice = None
self.task_server = None
self.nodes_manager_client = None
def enqueue_new_task(self, task):
# FIXME: remove after the new interface has been integrated with
if isinstance(task, dict):
task = self.task_server.task_manager.create_task(task)
else:
task.header.max_price = int(task.header.max_price)
resource_manager = self.resource_server.resource_manager
task_manager = self.task_server.task_manager
task_id = task.header.task_id
key_id = self.keys_auth.key_id
options = resource_manager.build_client_options(key_id)
files = task.get_resources(None, resource_types["hashes"])
deferred = self.resource_server.add_task(files, task_id, options)
deferred.addCallback(lambda _: task_manager.add_new_task(task))
return task
def task_resource_send(self, task_id):
self.task_server.task_manager.resources_send(task_id)
def task_resource_collected(self, task_id, unpack_delta=True):
self.task_server.task_computer.task_resource_collected(
task_id, unpack_delta)
def task_resource_failure(self, task_id, reason):
self.task_server.task_computer.task_resource_failure(task_id, reason)
def set_resource_port(self, resource_port):
self.resource_port = resource_port
self.p2pservice.set_resource_peer(self.node.prv_addr,
self.resource_port)
def run_test_task(self, t_dict):
if self.task_tester is None:
request = AsyncRequest(self._run_test_task, t_dict)
async_run(request)
return True
if self.rpc_publisher:
self.rpc_publisher.publish(Task.evt_task_test_status,
TaskTestStatus.error,
u"Another test is running")
return False
def _run_test_task(self, t_dict):
def on_success(*args, **kwargs):
self.task_tester = None
self._publish(Task.evt_task_test_status, TaskTestStatus.success,
*args, **kwargs)
def on_error(*args, **kwargs):
self.task_tester = None
self._publish(Task.evt_task_test_status, TaskTestStatus.error,
*args, **kwargs)
t = DictSerializer.load(t_dict)
self.task_tester = TaskTester(t, self.datadir, on_success, on_error)
self.task_tester.run()
self._publish(Task.evt_task_test_status, TaskTestStatus.started, True)
def abort_test_task(self):
with self.lock:
if self.task_tester is not None:
self.task_tester.end_comp()
return True
return False
def create_task(self, t_dict):
try:
task = DictSerializer.load(t_dict)
new_task = self.enqueue_new_task(task)
return unicode(new_task.header.task_id)
except Exception:
log.exception("Cannot create task {}".format(t_dict))
def abort_task(self, task_id):
self.task_server.task_manager.abort_task(task_id)
def restart_task(self, task_id):
self.task_server.task_manager.restart_task(task_id)
def restart_subtask(self, subtask_id):
self.task_server.task_manager.restart_subtask(subtask_id)
def pause_task(self, task_id):
self.task_server.task_manager.pause_task(task_id)
def resume_task(self, task_id):
self.task_server.task_manager.resume_task(task_id)
def delete_task(self, task_id):
self.remove_task_header(task_id)
self.remove_task(task_id)
self.task_server.task_manager.delete_task(task_id)
def get_node(self):
return DictSerializer.dump(self.node)
def get_node_name(self):
name = self.config_desc.node_name
return unicode(name) if name else u''
def get_neighbours_degree(self):
return self.p2pservice.get_peers_degree()
def get_suggested_addr(self, key_id):
return self.p2pservice.suggested_address.get(key_id)
def get_suggested_conn_reverse(self, key_id):
return self.p2pservice.get_suggested_conn_reverse(key_id)
def get_resource_peers(self):
self.p2pservice.send_get_resource_peers()
def get_peers(self):
return self.p2pservice.peers.values()
def get_known_peers(self):
peers = self.p2pservice.free_peers or []
return [
DictSerializer.dump(PeerSessionInfo(p), typed=False) for p in peers
]
def get_connected_peers(self):
peers = self.get_peers() or []
return [
DictSerializer.dump(PeerSessionInfo(p), typed=False) for p in peers
]
def get_public_key(self):
return self.keys_auth.public_key
def get_dir_manager(self):
if self.task_server:
return self.task_server.task_computer.dir_manager
def load_keys_from_file(self, file_name):
if file_name != "":
return self.keys_auth.load_from_file(file_name)
return False
def save_keys_to_files(self, private_key_path, public_key_path):
return self.keys_auth.save_to_files(private_key_path, public_key_path)
def get_key_id(self):
return self.get_client_id()
def get_difficulty(self):
return self.keys_auth.get_difficulty()
def get_client_id(self):
key_id = self.keys_auth.get_key_id()
return unicode(key_id) if key_id else None
def get_node_key(self):
key = self.node.key
return unicode(key) if key else None
def get_settings(self):
return DictSerializer.dump(self.config_desc)
def get_setting(self, key):
if not hasattr(self.config_desc, key):
raise KeyError(u"Unknown setting: {}".format(key))
value = getattr(self.config_desc, key)
if key in ConfigApprover.numeric_opt:
return unicode(value)
return value
def update_setting(self, key, value):
if not hasattr(self.config_desc, key):
raise KeyError(u"Unknown setting: {}".format(key))
setattr(self.config_desc, key, value)
self.change_config(self.config_desc)
def update_settings(self, settings_dict, run_benchmarks=False):
cfg_desc = DictSerializer.load(settings_dict)
self.change_config(cfg_desc, run_benchmarks)
def get_datadir(self):
return unicode(self.datadir)
def get_p2p_port(self):
return self.p2pservice.cur_port
def get_task_server_port(self):
return self.task_server.cur_port
def get_task_count(self):
return len(self.task_server.task_keeper.get_all_tasks())
def get_task(self, task_id):
return self.task_server.task_manager.get_task_dict(task_id)
def get_tasks(self, task_id=None):
if task_id:
return self.task_server.task_manager.get_task_dict(task_id)
return self.task_server.task_manager.get_tasks_dict()
def get_subtasks(self, task_id):
return self.task_server.task_manager.get_subtasks_dict(task_id)
def get_subtasks_borders(self, task_id):
return self.task_server.task_manager.get_subtasks_borders(task_id)
def get_subtask(self, subtask_id):
return self.task_server.task_manager.get_subtask_dict(subtask_id)
def get_task_preview(self, task_id, single=False):
return self.task_server.task_manager.get_task_preview(task_id,
single=single)
def get_task_stats(self):
return {
u'in_network': self.get_task_count(),
u'supported': self.get_supported_task_count(),
u'subtasks_computed': self.get_computed_task_count(),
u'subtasks_with_errors': self.get_error_task_count(),
u'subtasks_with_timeout': self.get_timeout_task_count()
}
def get_supported_task_count(self):
return len(self.task_server.task_keeper.supported_tasks)
def get_computed_task_count(self):
return self.task_server.task_computer.stats.get_stats('computed_tasks')
def get_timeout_task_count(self):
return self.task_server\
.task_computer.stats.get_stats('tasks_with_timeout')
def get_error_task_count(self):
return self.task_server\
.task_computer.stats.get_stats('tasks_with_errors')
def get_payment_address(self):
address = self.transaction_system.get_payment_address()
return unicode(address) if address else None
@inlineCallbacks
def get_balance(self):
if self.use_transaction_system():
req = AsyncRequest(self.transaction_system.get_balance)
b, ab, d = yield async_run(req)
if b is not None:
returnValue((unicode(b), unicode(ab), unicode(d)))
returnValue((None, None, None))
def get_payments_list(self):
if self.use_transaction_system():
payments = self.transaction_system.get_payments_list()
return map(self._map_payment, payments)
return ()
def get_incomes_list(self):
# Will be implemented in incomes_core
return []
@classmethod
def _map_payment(cls, obj):
obj["payee"] = to_unicode(obj["payee"])
obj["value"] = to_unicode(obj["value"])
obj["fee"] = to_unicode(obj["fee"])
return obj
def get_task_cost(self, task_id):
"""
Get current cost of the task defined by @task_id
:param task_id: Task ID
:return: Cost of the task
"""
cost = self.task_server.task_manager.get_payment_for_task_id(task_id)
if cost is None:
return 0.0
return cost
def use_transaction_system(self):
return bool(self.transaction_system)
def get_computing_trust(self, node_id):
if self.use_ranking():
return self.ranking.get_computing_trust(node_id)
return None
def get_requesting_trust(self, node_id):
if self.use_ranking():
return self.ranking.get_requesting_trust(node_id)
return None
def get_description(self):
try:
account, _ = Account.get_or_create(node_id=self.get_client_id())
return account.description
except Exception as e:
return u"An error has occurred {}".format(e)
def change_description(self, description):
self.get_description()
q = Account.update(description=description)\
.where(Account.node_id == self.get_client_id())
q.execute()
def use_ranking(self):
return bool(self.ranking)
def want_to_start_task_session(self, key_id, node_id, conn_id):
self.p2pservice.want_to_start_task_session(key_id, node_id, conn_id)
def inform_about_task_nat_hole(self, key_id, rv_key_id, addr, port,
ans_conn_id):
self.p2pservice.inform_about_task_nat_hole(key_id, rv_key_id, addr,
port, ans_conn_id)
def inform_about_nat_traverse_failure(self, key_id, res_key_id, conn_id):
self.p2pservice.inform_about_nat_traverse_failure(
key_id, res_key_id, conn_id)
# CLIENT CONFIGURATION
def set_rpc_server(self, rpc_server):
self.rpc_server = rpc_server
return self.rpc_server.add_service(self)
def change_config(self, new_config_desc, run_benchmarks=False):
self.config_desc = self.config_approver.change_config(new_config_desc)
self.cfg.change_config(self.config_desc)
self.p2pservice.change_config(self.config_desc)
self.upsert_hw_preset(HardwarePresets.from_config(self.config_desc))
if self.task_server:
self.task_server.change_config(self.config_desc,
run_benchmarks=run_benchmarks)
dispatcher.send(signal='golem.monitor',
event='config_update',
meta_data=self.__get_nodemetadatamodel())
def register_nodes_manager_client(self, nodes_manager_client):
self.nodes_manager_client = nodes_manager_client
def change_timeouts(self, task_id, full_task_timeout, subtask_timeout):
self.task_server.change_timeouts(task_id, full_task_timeout,
subtask_timeout)
def query_task_state(self, task_id):
state = self.task_server.task_manager.query_task_state(task_id)
if state:
return DictSerializer.dump(state)
def pull_resources(self, task_id, resources, client_options=None):
self.resource_server.download_resources(resources,
task_id,
client_options=client_options)
def add_resource_peer(self, node_name, addr, port, key_id, node_info):
self.resource_server.add_resource_peer(node_name, addr, port, key_id,
node_info)
def get_res_dirs(self):
return {
u"computing": self.get_computed_files_dir(),
u"received": self.get_received_files_dir(),
u"distributed": self.get_distributed_files_dir()
}
def get_res_dirs_sizes(self):
return {
unicode(name): unicode(du(d))
for name, d in self.get_res_dirs().iteritems()
}
def get_res_dir(self, dir_type):
if dir_type == DirectoryType.COMPUTED:
return self.get_computed_files_dir()
elif dir_type == DirectoryType.DISTRIBUTED:
return self.get_distributed_files_dir()
elif dir_type == DirectoryType.RECEIVED:
return self.get_received_files_dir()
raise Exception(u"Unknown dir type: {}".format(dir_type))
def get_computed_files_dir(self):
return unicode(self.task_server.get_task_computer_root())
def get_received_files_dir(self):
return unicode(self.task_server.task_manager.get_task_manager_root())
def get_distributed_files_dir(self):
return unicode(self.resource_server.get_distributed_resource_root())
def clear_dir(self, dir_type):
if dir_type == DirectoryType.COMPUTED:
return self.remove_computed_files()
elif dir_type == DirectoryType.DISTRIBUTED:
return self.remove_distributed_files()
elif dir_type == DirectoryType.RECEIVED:
return self.remove_received_files()
raise Exception(u"Unknown dir type: {}".format(dir_type))
def remove_computed_files(self):
dir_manager = DirManager(self.datadir)
dir_manager.clear_dir(self.get_computed_files_dir())
def remove_distributed_files(self):
dir_manager = DirManager(self.datadir)
dir_manager.clear_dir(self.get_distributed_files_dir())
def remove_received_files(self):
dir_manager = DirManager(self.datadir)
dir_manager.clear_dir(self.get_received_files_dir())
def remove_task(self, task_id):
self.p2pservice.remove_task(task_id)
def remove_task_header(self, task_id):
self.task_server.remove_task_header(task_id)
def get_known_tasks(self):
headers = {}
for key, header in self.task_server.task_keeper.task_headers.iteritems(
): # noqa
headers[unicode(key)] = DictSerializer.dump(header)
return headers
def get_environments(self):
envs = copy(self.environments_manager.get_environments())
return [{
u'id': unicode(env.get_id()),
u'supported': env.supported(),
u'accepted': env.is_accepted(),
u'performance': env.get_performance(self.config_desc),
u'description': unicode(env.short_description)
} for env in envs]
@inlineCallbacks
def run_benchmark(self, env_id):
# TODO: move benchmarks to environments
from apps.blender.blenderenvironment import BlenderEnvironment
from apps.lux.luxenvironment import LuxRenderEnvironment
deferred = Deferred()
if env_id == BlenderEnvironment.get_id():
self.task_server.task_computer.run_blender_benchmark(
deferred.callback, deferred.errback)
elif env_id == LuxRenderEnvironment.get_id():
self.task_server.task_computer.run_lux_benchmark(
deferred.callback, deferred.errback)
else:
raise Exception("Unknown environment: {}".format(env_id))
result = yield deferred
returnValue(result)
def enable_environment(self, env_id):
self.environments_manager.change_accept_tasks(env_id, True)
def disable_environment(self, env_id):
self.environments_manager.change_accept_tasks(env_id, False)
def send_gossip(self, gossip, send_to):
return self.p2pservice.send_gossip(gossip, send_to)
def send_stop_gossip(self):
return self.p2pservice.send_stop_gossip()
def collect_gossip(self):
return self.p2pservice.pop_gossips()
def collect_stopped_peers(self):
return self.p2pservice.pop_stop_gossip_form_peers()
def collect_neighbours_loc_ranks(self):
return self.p2pservice.pop_neighbours_loc_ranks()
def push_local_rank(self, node_id, loc_rank):
self.p2pservice.push_local_rank(node_id, loc_rank)
def check_payments(self):
if not self.transaction_system:
return
after_deadline_nodes = self.transaction_system.check_payments()
for node_id in after_deadline_nodes:
Trust.PAYMENT.decrease(node_id)
@staticmethod
def save_task_preset(preset_name, task_type, data):
taskpreset.save_task_preset(preset_name, task_type, data)
@staticmethod
def get_task_presets(task_type):
log.info("Loading presets for {}".format(task_type))
return taskpreset.get_task_presets(task_type)
@staticmethod
def delete_task_preset(task_type, preset_name):
taskpreset.delete_task_preset(task_type, preset_name)
def _publish(self, event_name, *args, **kwargs):
if self.rpc_publisher:
self.rpc_publisher.publish(event_name, *args, **kwargs)
def lock_config(self, on=True):
self._publish(UI.evt_lock_config, on)
def config_changed(self):
self._publish(Environment.evt_opts_changed)
def __get_nodemetadatamodel(self):
return NodeMetadataModel(self.get_client_id(), self.session_id,
sys.platform, APP_VERSION,
self.get_description(), self.config_desc)
def __try_to_change_to_number(self,
old_value,
new_value,
to_int=False,
to_float=False,
name="Config"):
try:
if to_int:
new_value = int(new_value)
elif to_float:
new_value = float(new_value)
except ValueError:
log.warning("%s value '%s' is not a number", name, new_value)
new_value = old_value
return new_value
def __do_work(self):
if not self.p2pservice:
return
if self.config_desc.send_pings:
self.p2pservice.ping_peers(self.config_desc.pings_interval)
try:
self.p2pservice.sync_network()
except Exception:
log.exception("p2pservice.sync_network failed")
try:
self.task_server.sync_network()
except Exception:
log.exception("task_server.sync_network failed")
try:
self.resource_server.sync_network()
except Exception:
log.exception("resource_server.sync_network failed")
try:
self.ranking.sync_network()
except Exception:
log.exception("ranking.sync_network failed")
try:
self.check_payments()
except Exception:
log.exception("check_payments failed")
@inlineCallbacks
def __publish_events(self):
now = time.time()
delta = now - self.last_nss_time
if delta > max(self.config_desc.node_snapshot_interval, 1):
dispatcher.send(
signal='golem.monitor',
event='stats_snapshot',
known_tasks=self.get_task_count(),
supported_tasks=self.get_supported_task_count(),
stats=self.task_server.task_computer.stats,
)
dispatcher.send(
signal='golem.monitor',
event='task_computer_snapshot',
task_computer=self.task_server.task_computer,
)
# with self.snapshot_lock:
# self.__make_node_state_snapshot()
# self.manager_server.sendStateMessage(self.last_node_state_snapshot)
self.last_nss_time = time.time()
delta = now - self.last_net_check_time
if delta >= self.config_desc.network_check_interval:
self.last_net_check_time = time.time()
self._publish(Network.evt_connection, self.connection_status())
if now - self.last_tasks_time >= PUBLISH_TASKS_INTERVAL:
self._publish(Task.evt_task_list, self.get_tasks())
if now - self.last_balance_time >= PUBLISH_BALANCE_INTERVAL:
try:
gnt, av_gnt, eth = yield self.get_balance()
except Exception as exc:
log.debug('Error retrieving balance: {}'.format(exc))
else:
self._publish(
Payments.evt_balance, {
u'GNT': unicode(gnt),
u'GNT_available': unicode(av_gnt),
u'ETH': unicode(eth)
})
def __make_node_state_snapshot(self, is_running=True):
peers_num = len(self.p2pservice.peers)
last_network_messages = self.p2pservice.get_last_messages()
if self.task_server:
tasks_num = len(self.task_server.task_keeper.task_headers)
r_tasks_progs = self.task_server.task_computer.get_progresses()
l_tasks_progs = self.task_server.task_manager.get_progresses()
last_task_messages = self.task_server.get_last_messages()
self.last_node_state_snapshot = NodeStateSnapshot(
is_running, self.config_desc.node_name, peers_num, tasks_num,
self.p2pservice.node.pub_addr, self.p2pservice.node.pub_port,
last_network_messages, last_task_messages, r_tasks_progs,
l_tasks_progs)
else:
self.last_node_state_snapshot = NodeStateSnapshot(
self.config_desc.node_name, peers_num)
if self.nodes_manager_client:
self.nodes_manager_client.send_client_state_snapshot(
self.last_node_state_snapshot)
def connection_status(self):
listen_port = self.get_p2p_port()
task_server_port = self.get_task_server_port()
if listen_port == 0 or task_server_port == 0:
return u"Application not listening, check config file."
messages = []
if self.node.port_status:
statuses = self.node.port_status.split('\n')
failures = filter(lambda e: e.find('open') == -1, statuses)
messages.append(u"Port " + u", ".join(failures) + u".")
if self.get_connected_peers():
messages.append(u"Connected")
else:
messages.append(u"Not connected to Golem Network, "
u"check seed parameters.")
return u' '.join(messages)
def get_metadata(self):
metadata = dict()
# if self.ipfs_manager:
# metadata.update(self.ipfs_manager.get_metadata())
return metadata
def interpret_metadata(self, metadata, address, port, node_info):
pass
# if self.config_desc and node_info and metadata:
# seed_addresses = self.p2pservice.get_seeds()
# node_addresses = [
# (address, port),
# (node_info.pub_addr, node_info.pub_port)
# ]
# self.ipfs_manager.interpret_metadata(metadata,
# seed_addresses,
# node_addresses)
def get_status(self):
progress = self.task_server.task_computer.get_progresses()
if len(progress) > 0:
msg = u"Computing {} subtask(s):".format(len(progress))
for k, v in progress.iteritems():
msg = u"{} \n {} ({}%)\n".format(msg, k,
v.get_progress() * 100)
elif self.config_desc.accept_tasks:
msg = u"Waiting for tasks...\n"
else:
msg = u"Not accepting tasks\n"
peers = self.p2pservice.get_peers()
msg += u"Active peers in network: {}\n".format(len(peers))
return msg
def activate_hw_preset(self, name, run_benchmarks=False):
HardwarePresets.update_config(name, self.config_desc)
if hasattr(self, 'task_server') and self.task_server:
self.task_server.change_config(self.config_desc,
run_benchmarks=run_benchmarks)
def __lock_datadir(self):
if not path.exists(self.datadir):
# Create datadir if not exists yet.
# TODO: It looks we have the same code in many places
makedirs(self.datadir)
self.__datadir_lock = open(path.join(self.datadir, "LOCK"), 'w')
flags = filelock.LOCK_EX | filelock.LOCK_NB
try:
filelock.lock(self.__datadir_lock, flags)
except IOError:
raise IOError("Data dir {} used by other Golem instance".format(
self.datadir))
def _unlock_datadir(self):
# solves locking issues on OS X
try:
filelock.unlock(self.__datadir_lock)
except Exception:
pass
self.__datadir_lock.close()