def __init__(self, config_desc, keys_auth, client, use_ipv6=False):
self.client = client
self.keys_auth = keys_auth
self.resources_to_send = []
self.resources_to_get = []
self.res_send_it = 0
self.peers_it = 0
self.dir_manager = DirManager(client.datadir)
self.resource_manager = DistributedResourceManager(
self.dir_manager.get_resource_dir())
self.use_ipv6 = use_ipv6
network = TCPNetwork(
ProtocolFactory(FilesProtocol, self,
SessionFactory(ResourceSession)), use_ipv6)
PendingConnectionsServer.__init__(self, config_desc, network)
self.resource_peers = {}
self.waiting_tasks = {}
self.waiting_tasks_to_compute = {}
self.waiting_resources = {}
self.last_get_resource_peers_time = time.time()
self.get_resource_peers_interval = 5.0
self.sessions = []
self.last_message_time_threshold = config_desc.resource_session_timeout
def change_config(self, config_desc, run_benchmarks=False):
PendingConnectionsServer.change_config(self, config_desc)
self.config_desc = config_desc
self.last_message_time_threshold = config_desc.task_session_timeout
self.task_manager.change_config(
self.__get_task_manager_root(self.client.datadir),
config_desc.use_distributed_resource_management)
self.task_computer.change_config(config_desc,
run_benchmarks=run_benchmarks)
self.task_keeper.change_config(config_desc)
def __init__(self,
node,
config_desc,
keys_auth,
client,
use_ipv6=False,
use_docker_machine_manager=True):
self.client = client
self.keys_auth = keys_auth
self.config_desc = config_desc
self.node = node
self.task_keeper = TaskHeaderKeeper(client.environments_manager,
min_price=config_desc.min_price)
self.task_manager = TaskManager(
config_desc.node_name,
self.node,
self.keys_auth,
root_path=TaskServer.__get_task_manager_root(client.datadir),
use_distributed_resources=config_desc.
use_distributed_resource_management,
tasks_dir=os.path.join(client.datadir, 'tasks'))
self.task_computer = TaskComputer(
config_desc.node_name,
task_server=self,
use_docker_machine_manager=use_docker_machine_manager)
self.task_connections_helper = TaskConnectionsHelper()
self.task_connections_helper.task_server = self
self.task_sessions = {}
self.task_sessions_incoming = WeakList()
self.max_trust = 1.0
self.min_trust = 0.0
self.last_messages = []
self.last_message_time_threshold = config_desc.task_session_timeout
self.results_to_send = {}
self.failures_to_send = {}
self.use_ipv6 = use_ipv6
self.forwarded_session_request_timeout = config_desc.waiting_for_task_session_timeout
self.forwarded_session_requests = {}
self.response_list = {}
self.deny_set = get_deny_set(datadir=client.datadir)
network = TCPNetwork(
ProtocolFactory(MidAndFilesProtocol, self,
SessionFactory(TaskSession)), use_ipv6)
PendingConnectionsServer.__init__(self, config_desc, network)
def test_get_socket_addresses(self):
server = PendingConnectionsServer(None, Network())
node = Node()
port = 100
res = server.get_socket_addresses(node, port, None)
self.assertEqual(res, [])
node.pub_addr = "10.10.10.10"
res = server.get_socket_addresses(node, port, None)
self.assertEqual(len(res), 1)
self.assertEqual(res[0].address, node.pub_addr)
self.assertEqual(res[0].port, port)
node.pub_port = 1023
res = server.get_socket_addresses(node, port, None)
self.assertEqual(len(res), 1)
self.assertEqual(res[0].address, node.pub_addr)
self.assertEqual(res[0].port, 1023)
node.prv_addresses = [
"10.10.10.1", "10.10.10.2", "10.10.10.3", "10.10.10.4"
]
res = server.get_socket_addresses(node, port, None)
self.assertEqual(len(res), 5)
self.assertEqual(res[4].address, node.pub_addr)
self.assertEqual(res[4].port, 1023)
for i in range(4):
self.assertEqual(res[i].address, node.prv_addresses[i])
self.assertEqual(res[i].port, port)
node.pub_addr = None
res = server.get_socket_addresses(node, port, None)
self.assertEqual(len(res), 4)
for i in range(4):
self.assertEqual(res[i].address, node.prv_addresses[i])
self.assertEqual(res[i].port, port)
def get_socket_addresses(self, node_info, port, key_id):
if self.client.get_suggested_conn_reverse(key_id):
return []
socket_addresses = PendingConnectionsServer.get_socket_addresses(
self, node_info, port, key_id)
addr = self.client.get_suggested_addr(key_id)
if addr:
socket_addresses = [SocketAddress(addr, port)] + socket_addresses
return socket_addresses
def test_get_socket_addresses(self):
server = PendingConnectionsServer(None, Network())
node = self.node_info
res = server.get_socket_addresses(node,
prv_port=node.prv_port,
pub_port=node.pub_port)
self.assertEqual(res, [
SocketAddress(self.node_info.pub_addr, self.node_info.pub_port),
SocketAddress(self.node_info.prv_addresses[0],
self.node_info.prv_port)
])
node.pub_addr = "10.10.10.10"
res = server.get_socket_addresses(node,
prv_port=node.prv_port,
pub_port=node.pub_port)
self.assertEqual(len(res), 2)
self.assertEqual(res[0].address, node.pub_addr)
self.assertEqual(res[0].port, node.pub_port)
node.pub_port = 1023
res = server.get_socket_addresses(node,
prv_port=node.prv_port,
pub_port=node.pub_port)
self.assertEqual(len(res), 2)
self.assertEqual(res[0].address, node.pub_addr)
self.assertEqual(res[0].port, 1023)
node.prv_addresses = [
"10.10.10.1", "10.10.10.2", "10.10.10.3", "10.10.10.4"
]
res = server.get_socket_addresses(node,
prv_port=node.prv_port,
pub_port=node.pub_port)
self.assertEqual(len(res), 5)
self.assertEqual(res[4].address, node.prv_addresses[-1])
self.assertEqual(res[4].port, 1233)
for i in range(4):
self.assertEqual(res[i + 1].address, node.prv_addresses[i])
self.assertEqual(res[i + 1].port, node.prv_port)
node.pub_addr = None
res = server.get_socket_addresses(node,
prv_port=node.prv_port,
pub_port=node.pub_port)
self.assertEqual(len(res), 4)
for i in range(4):
self.assertEqual(res[i].address, node.prv_addresses[i])
self.assertEqual(res[i].port, node.prv_port)
def test_address_accessible(self):
config = Mock()
config.use_ipv6 = False
server = PendingConnectionsServer(config, Mock())
assert not server._is_address_accessible(None)
sockv4 = SocketAddress('8.8.8.8', 40100)
sockv6 = SocketAddress('2001:0db8:85a3:0000:0000:8a2e:abcd:efea',
40100)
assert server._is_address_accessible(sockv4)
assert not server._is_address_accessible(sockv6)
server.use_ipv6 = True
assert server._is_address_accessible(sockv4)
assert server._is_address_accessible(sockv6)
def __init__(self,
node,
config_desc: ClientConfigDescriptor,
client,
use_ipv6=False,
use_docker_manager=True,
task_archiver=None,
apps_manager=AppsManager(),
task_finished_cb=None) -> None:
self.client = client
self.keys_auth = client.keys_auth
self.config_desc = config_desc
self.node = node
self.task_archiver = task_archiver
self.task_keeper = TaskHeaderKeeper(
environments_manager=client.environments_manager,
node=self.node,
min_price=config_desc.min_price,
task_archiver=task_archiver)
self.task_manager = TaskManager(
config_desc.node_name,
self.node,
self.keys_auth,
root_path=TaskServer.__get_task_manager_root(client.datadir),
use_distributed_resources=config_desc.
use_distributed_resource_management,
tasks_dir=os.path.join(client.datadir, 'tasks'),
apps_manager=apps_manager,
finished_cb=task_finished_cb,
)
benchmarks = self.task_manager.apps_manager.get_benchmarks()
self.benchmark_manager = BenchmarkManager(
node_name=config_desc.node_name,
task_server=self,
root_path=self.get_task_computer_root(),
benchmarks=benchmarks)
self.task_computer = TaskComputer(
task_server=self,
use_docker_manager=use_docker_manager,
finished_cb=task_finished_cb)
self.task_connections_helper = TaskConnectionsHelper()
self.task_connections_helper.task_server = self
self.task_sessions = {}
self.task_sessions_incoming = weakref.WeakSet()
self.max_trust = 1.0
self.min_trust = 0.0
self.last_messages = []
self.last_message_time_threshold = config_desc.task_session_timeout
self.results_to_send = {}
self.failures_to_send = {}
self.use_ipv6 = use_ipv6
self.forwarded_session_request_timeout = \
config_desc.waiting_for_task_session_timeout
self.forwarded_session_requests = {}
self.response_list = {}
self.acl = get_acl(Path(client.datadir))
self.resource_handshakes = {}
network = TCPNetwork(
ProtocolFactory(SafeProtocol, self, SessionFactory(TaskSession)),
use_ipv6)
PendingConnectionsServer.__init__(self, config_desc, network)
# instantiate ReceivedMessageHandler connected to self
# to register in golem.network.concent.handlers_library
from golem.network.concent import \
received_handler as concent_received_handler
self.concent_handler = \
concent_received_handler.TaskServerMessageHandler(self)
dispatcher.connect(self.income_listener, signal='golem.income')
dispatcher.connect(self.finished_task_listener,
signal='golem.taskmanager')
def test_sync_pending(self):
network = Network()
server = PendingConnectionsServer(None, network)
req_type = 0
final_failure_called = [False]
node_info = Mock(key='1234',
prv_addresses=["1.2.3.4"],
pub_addr="1.2.3.4",
pub_port=self.port)
def final_failure(*args, **kwargs):
final_failure_called[0] = True
server.conn_established_for_type[req_type] = lambda x: x
server.conn_failure_for_type[req_type] = server.final_conn_failure
server.conn_final_failure_for_type[req_type] = final_failure
server._is_address_accessible = Mock(return_value=True)
server._add_pending_request(req_type,
node_info,
prv_port=node_info.prv_port,
pub_port=node_info.pub_port,
args={})
assert len(server.pending_connections) == 1
server._sync_pending()
assert network.connected
network.connected = False
server.pending_connections = {}
server._add_pending_request(req_type,
self.node_info,
prv_port=self.node_info.prv_port,
pub_port=self.node_info.pub_port,
args={})
assert len(server.pending_connections) == 1
pending_conn = next(iter(list(server.pending_connections.values())))
pending_conn.socket_addresses = []
server._sync_pending()
assert not network.connected
assert final_failure_called[0]
def test_pending_conn(self):
network = Network()
server = PendingConnectionsServer(None, network)
req_type = 0
final_failure_called = [False]
def final_failure(*args, **kwargs):
final_failure_called[0] = True
server.conn_established_for_type[req_type] = lambda x: x
server.conn_failure_for_type[req_type] = server.final_conn_failure
server.conn_final_failure_for_type[req_type] = final_failure
server._is_address_accessible = Mock(return_value=True)
server._add_pending_request(req_type,
self.node_info,
prv_port=self.node_info.prv_port,
pub_port=self.node_info.pub_port,
args={})
assert len(server.pending_connections) == 1
pending_conn = next(iter(list(server.pending_connections.values())))
final_failure_called[0] = False
server.final_conn_failure(pending_conn.id)
assert final_failure_called[0]
server.verified_conn(pending_conn.id)
assert len(server.pending_connections) == 0
final_failure_called[0] = False
server.final_conn_failure(pending_conn.id)
assert not final_failure_called[0]
server._add_pending_request(req_type,
self.node_info,
prv_port=self.node_info.prv_port,
pub_port=self.node_info.pub_port,
args={})
pending_conn = next(iter(list(server.pending_connections.values())))
server._mark_connected(pending_conn.id, "10.10.10.1", self.port)
assert pending_conn.status == PenConnStatus.Connected
assert SocketAddress("10.10.10.1",
self.port) == pending_conn.socket_addresses[0]
def start_accepting(self):
PendingConnectionsServer.start_accepting(self)
def test_sync_listen(self):
network = Network()
server = PendingConnectionsServer(None, network)
req_type = 0
server.listen_established_for_type[req_type] = lambda x: x
server.listen_failure_for_type[req_type] = server.final_conn_failure
server._add_pending_listening(req_type, self.port, {})
assert len(server.pending_listenings) == 1
pending_lis = server.pending_listenings[0]
pending_lis.time = 0
server._sync_pending()
assert len(server.pending_listenings) == 0
assert len(server.open_listenings) == 1
server.last_check_listening_time = 0
server._remove_old_listenings()
assert network.stop_listening_called
assert len(server.pending_listenings) == 0
assert len(server.open_listenings) == 0
def test_sync_pending(self):
network = Network()
server = PendingConnectionsServer(None, network)
req_type = 0
final_failure_called = [False]
node_info = Mock()
node_info.prv_addresses = ["1.2.3.4"]
node_info.pub_addr = "1.2.3.4"
node_info.pub_port = self.port
def final_failure(_):
final_failure_called[0] = True
server.conn_established_for_type[req_type] = lambda x: x
server.conn_failure_for_type[req_type] = server.final_conn_failure
server.conn_final_failure_for_type[req_type] = final_failure
server._add_pending_request(req_type,
node_info,
self.port,
self.key_id,
args={})
assert len(server.pending_connections) == 1
server._sync_pending()
assert network.connected
network.connected = False
server.pending_connections = {}
server._add_pending_request(req_type,
node_info,
self.port,
self.key_id,
args={})
assert len(server.pending_connections) == 1
pending_conn = next(server.pending_connections.itervalues())
pending_conn.socket_addresses = []
server._sync_pending()
assert not network.connected
assert final_failure_called[0]