def get_socket_addresses(self, node_info, prv_port=None, pub_port=None):
""" Change node info into tcp addresses. Adds a suggested address.
:param Node node_info: node information
:param prv_port: private port that should be used
:param pub_port: public port that should be used
:return:
"""
prv_port = prv_port or node_info.prv_port
pub_port = pub_port or node_info.pub_port
socket_addresses = super().get_socket_addresses(node_info=node_info,
prv_port=prv_port,
pub_port=pub_port)
address = self.client.get_suggested_addr(node_info.key)
if not address:
return socket_addresses
if self._is_address_valid(address, prv_port):
socket_address = SocketAddress(address, prv_port)
self._prepend_address(socket_addresses, socket_address)
if self._is_address_valid(address, pub_port):
socket_address = SocketAddress(address, pub_port)
self._prepend_address(socket_addresses, socket_address)
return socket_addresses[:MAX_CONNECT_SOCKET_ADDRESSES]
def test_get_socket_addresses(self):
key_id = 'abcd'
address = '10.0.0.1'
prv_port = 1
pub_port = 2
node_info = mock.Mock(key=key_id)
self.service.suggested_address[key_id] = address
result = self.service.get_socket_addresses(node_info, prv_port,
pub_port)
assert SocketAddress(address, prv_port) in result
assert SocketAddress(address, pub_port) in result
def __expect_exception(self, value, exception):
try:
SocketAddress.parse(value)
suffix = ', not succeed'
except exception:
return
except:
exc = sys.exc_info()[0]
suffix = ', not ' + exc.__name__
self.fail('SocketAddress.parse("' + str(value) + '") should raise ' +
exception.__name__ + suffix)
def parse_http_addr(ctx, param, value):
del ctx, param
if value:
try:
http_prefix = 'https://'
if not value.startswith(http_prefix):
raise click.BadParameter("Address without https:// prefix"
"specified: {}".format(value))
SocketAddress.parse(value[len(http_prefix):])
return value
except ipaddress.AddressValueError as e:
raise click.BadParameter(
"Invalid network address specified: {}".format(e))
return None
def test_get_socket_addresses(self):
server = PendingConnectionsServer(None, Network())
node = self.node_info
res = server.get_socket_addresses(node,
prv_port=node.prv_port,
pub_port=node.pub_port)
self.assertEqual(res, [
SocketAddress(self.node_info.pub_addr, self.node_info.pub_port),
SocketAddress(self.node_info.prv_addresses[0],
self.node_info.prv_port)
])
node.pub_addr = "10.10.10.10"
res = server.get_socket_addresses(node,
prv_port=node.prv_port,
pub_port=node.pub_port)
self.assertEqual(len(res), 2)
self.assertEqual(res[0].address, node.pub_addr)
self.assertEqual(res[0].port, node.pub_port)
node.pub_port = 1023
res = server.get_socket_addresses(node,
prv_port=node.prv_port,
pub_port=node.pub_port)
self.assertEqual(len(res), 2)
self.assertEqual(res[0].address, node.pub_addr)
self.assertEqual(res[0].port, 1023)
node.prv_addresses = [
"10.10.10.1", "10.10.10.2", "10.10.10.3", "10.10.10.4"
]
res = server.get_socket_addresses(node,
prv_port=node.prv_port,
pub_port=node.pub_port)
self.assertEqual(len(res), 5)
self.assertEqual(res[4].address, node.prv_addresses[-1])
self.assertEqual(res[4].port, 1233)
for i in range(4):
self.assertEqual(res[i + 1].address, node.prv_addresses[i])
self.assertEqual(res[i + 1].port, node.prv_port)
node.pub_addr = None
res = server.get_socket_addresses(node,
prv_port=node.prv_port,
pub_port=node.pub_port)
self.assertEqual(len(res), 4)
for i in range(4):
self.assertEqual(res[i].address, node.prv_addresses[i])
self.assertEqual(res[i].port, node.prv_port)
def connect(self, ip, port_):
try:
sa = SocketAddress(ip, int(port_))
Network.client.connect((sa.address, sa.port))
except Exception as exc:
return CommandResult(
error="Cannot connect to {}:{}: {}".format(ip, port_, exc))
def test_refresh_peers(self):
sa = SocketAddress('127.0.0.1', 11111)
node = mock.MagicMock()
node.key = encode_hex(urandom(64))[2:]
node.key_id = node.key
node.address = sa
node2 = mock.MagicMock()
node2.key = encode_hex(urandom(64))[2:]
node2.key_id = node2.key
node2.address = sa
self.service.add_peer(node)
self.service.add_peer(node2)
self.service.peers[node.key].last_message_time = time.time() + 1000
self.service.peers[node2.key].last_message_time = time.time() + 1000
self.service.config_desc.opt_peer_num = 1000
assert len(self.service.peers) == 2
self.service.sync_network()
assert len(self.service.peers) == 2
self.service.last_refresh_peers = 0
self.service.last_peers_request = 0
self.service._peer_dbg_time_threshold = 0
self.service.sync_network()
# disabled
assert len(self.service.peers) == 2
def test_run_computing_node(self, mock_reactor):
client = runner.run_computing_node(self.path,
SocketAddress("127.0.0.1", 40102))
environments = list(client.environments_manager.environments)
self.assertTrue(any(env.get_id() == task.DummyTask.ENVIRONMENT_NAME
for env in environments))
client.quit()
def test_refresh_peers(self):
sa = SocketAddress('127.0.0.1', 11111)
node = MagicMock()
node.key = EllipticalKeysAuth(self.path, "TESTPRIV", "TESTPUB").get_key_id()
node.key_id = node.key
node.address = sa
node2 = MagicMock()
node2.key = EllipticalKeysAuth(self.path, "TESTPRIV2", "TESTPUB2").get_key_id()
node2.key_id = node2.key
node2.address = sa
self.service.add_peer(node.key, node)
self.service.add_peer(node2.key, node2)
self.service.peers[node.key].last_message_time = time.time() + 1000
self.service.peers[node2.key].last_message_time = time.time() + 1000
self.service.config_desc.opt_peer_num = 1000
assert len(self.service.peers) == 2
self.service.sync_network()
assert len(self.service.peers) == 2
self.service.last_refresh_peers = 0
self.service.last_peers_request = 0
self.service._peer_dbg_time_threshold = 0
self.service.sync_network()
# disabled
assert len(self.service.peers) == 2
def test_many_peers(self, config_logging, mock_node):
addr1 = '10.30.10.216:40111'
addr2 = '10.30.10.214:3333'
runner = CliRunner()
args = self.args + ['--peer', addr1, '--peer', addr2]
return_value = runner.invoke(start, args, catch_exceptions=False)
self.assertEqual(return_value.exit_code, 0)
mock_node.assert_has_calls(
[call().run(use_rpc=True),
call().add_tasks([])], any_order=True)
call_names = [name for name, arg, kwarg in mock_node.mock_calls]
self.assertTrue('().connect_with_peers' in call_names)
peer_num = call_names.index('().connect_with_peers')
peer_arg = mock_node.mock_calls[peer_num][1][0]
self.assertEqual(len(peer_arg), 2)
self.assertEqual(peer_arg[0], SocketAddress.parse(addr1))
self.assertEqual(peer_arg[1], SocketAddress.parse(addr2))
def test_run_computing_node(self, mock_config_logging, mock_reactor, _):
client = runner.run_computing_node(self.path,
SocketAddress("127.0.0.1", 40102))
assert task.DummyTask.ENVIRONMENT_NAME in \
client.environments_manager.environments
self.assertTrue(mock_reactor.run.called)
self.assertTrue(mock_config_logging.called)
client.quit()
def connect(self, socket_address):
if isinstance(socket_address, Iterable):
socket_address = SocketAddress(socket_address[0],
int(socket_address[1]))
log.debug("P2pservice connecting to %s on port %s",
socket_address.address, socket_address.port)
self.p2pservice.connect(socket_address)
def parse_rpc_address(ctx, param, value):
del ctx, param
if value:
try:
return SocketAddress.parse(value)
except AddressValueError as e:
raise click.BadParameter(
"Invalid RPC address specified: {}".format(e.message))
def __expect_valid(self, value):
if value[0] == '[':
addr, port = value.split(']:')
addr = addr[1:]
else:
addr, port = value.split(':')
a = SocketAddress.parse(value)
self.assertTrue(a.address == addr)
self.assertTrue(a.port == int(port))
def traverse_nat(self, key_id, addr, port, conn_id, super_key_id):
connect_info = TCPConnectInfo(
[SocketAddress(addr, port)],
self.__connection_for_traverse_nat_established,
self.__connection_for_traverse_nat_failure)
self.network.connect(connect_info,
client_key_id=key_id,
conn_id=conn_id,
super_key_id=super_key_id)
def get_socket_addresses(self, node_info, port, key_id):
if self.client.get_suggested_conn_reverse(key_id):
return []
socket_addresses = PendingConnectionsServer.get_socket_addresses(
self, node_info, port, key_id)
addr = self.client.get_suggested_addr(key_id)
if addr:
socket_addresses = [SocketAddress(addr, port)] + socket_addresses
return socket_addresses
def parse_peer(ctx, param, value):
del ctx, param
addresses = []
for arg in value:
try:
addresses.append(SocketAddress.parse(arg))
except AddressValueError as e:
raise click.BadParameter(
"Invalid peer address specified: {}".format(e.message))
return addresses
def test_address_accessible(self):
config = Mock()
config.use_ipv6 = False
server = PendingConnectionsServer(config, Mock())
assert not server._is_address_accessible(None)
sockv4 = SocketAddress('8.8.8.8', 40100)
sockv6 = SocketAddress('2001:0db8:85a3:0000:0000:8a2e:abcd:efea',
40100)
assert server._is_address_accessible(sockv4)
assert not server._is_address_accessible(sockv6)
server.use_ipv6 = True
assert server._is_address_accessible(sockv4)
assert server._is_address_accessible(sockv6)
def parse_node_addr(ctx, param, value):
del ctx, param
if value:
try:
SocketAddress(value, 1)
return value
except AddressValueError as e:
raise click.BadParameter(
"Invalid network address specified: {}".format(e.message))
return ''
def parse_rpc_address(ctx, param, value):
del ctx, param
value = to_unicode(value)
if value:
try:
return SocketAddress.parse(value)
except ipaddress.AddressValueError as e:
raise click.BadParameter(
"Invalid RPC address specified: {}".format(e))
return None
def test_zone_index(self):
base_address = u"fe80::3"
address = u"fe80::3%eth0"
port = 1111
sa = SocketAddress(address, port)
assert sa.address == base_address
assert sa.port == port
address = u"fe80::3%1"
sa = SocketAddress(address, port)
assert sa.address == base_address
address = u"fe80::3%en0"
sa = SocketAddress(address, port)
assert sa.address == base_address
address = base_address
sa = SocketAddress(address, port)
assert sa.address == base_address
def check_rpc_address(ctx, param, address):
split = address.rsplit(':', 1)
host, port = split[0], int(split[1])
try:
SocketAddress(host, port)
except AddressValueError as e:
return click.BadParameter(
"Invalid network address specified: {}".format(e.message))
return WebSocketAddress(host, port, u'golem')
def test_connect_success(self, connection_established, createSocket):
createSocket.return_value = socket = mock.Mock()
socket.fileno = mock.Mock(return_value=0)
socket.getsockopt = mock.Mock(return_value=None)
socket.connect_ex = mock.Mock(return_value=EISCONN)
addr = SocketAddress('127.0.0.1', 40102)
self.service.connect(addr)
time.sleep(0.1)
assert connection_established.called
assert connection_established.call_args[0][0] is not None
assert connection_established.call_args[1]['conn_id'] is not None
def test_runner_dispatch_computing(
self, mock_run_simulation, mock_run_computing_node,
mock_run_requesting_node):
args = ["runner.py", runner.COMPUTING_NODE_KIND,
self.path, "1.2.3.4:5678"]
runner.dispatch(args)
self.assertFalse(mock_run_requesting_node.called)
self.assertTrue(mock_run_computing_node.called)
self.assertEqual(mock_run_computing_node.call_args[0],
(self.path, SocketAddress("1.2.3.4", 5678)))
self.assertEqual(mock_run_computing_node.call_args[1],
{"fail_after": None})
self.assertFalse(mock_run_simulation.called)
def test_runner_dispatch_computing_with_failure(
self, mock_run_simulation, mock_run_computing_node,
mock_run_requesting_node):
args = ["runner.py", runner.COMPUTING_NODE_KIND,
self.path, "10.0.255.127:16000", "25"]
runner.dispatch(args)
self.assertFalse(mock_run_requesting_node.called)
self.assertTrue(mock_run_computing_node.called)
self.assertEqual(mock_run_computing_node.call_args[0],
(self.path, SocketAddress("10.0.255.127", 16000)))
self.assertEqual(mock_run_computing_node.call_args[1],
{"fail_after": 25.0})
self.assertFalse(mock_run_simulation.called)
def __str__(self):
ip4, ip6 = 'ip4', 'ip6'
proto = self.proto or 'tcp'
port = self.port or IPFS_DEFAULT_TCP_PORT
sa = SocketAddress(self.ip_address, port)
if self.encap_proto:
return self.pattern_encap.format(ip6 if sa.ipv6 else ip4,
self.ip_address, proto, port,
self.encap_proto, self.node_id)
else:
return self.pattern.format(ip6 if sa.ipv6 else ip4,
self.ip_address, proto, port,
self.node_id)
def test_peers(self, config_logging, mock_node):
runner = CliRunner()
return_value = runner.invoke(
start,
self.args + [
'--peer', u'10.30.10.216:40111', u'--peer',
u'[2001:db8:85a3:8d3:1319:8a2e:370:7348]:443', '--peer',
'[::ffff:0:0:0]:96'
],
catch_exceptions=False)
self.assertEqual(return_value.exit_code, 0)
mock_node.assert_has_calls(
[call().run(use_rpc=True),
call().add_tasks([])], any_order=True)
call_names = [name for name, arg, kwarg in mock_node.mock_calls]
self.assertTrue('().connect_with_peers' in call_names)
peer_num = call_names.index('().connect_with_peers')
peer_arg = mock_node.mock_calls[peer_num][1][0]
self.assertEqual(len(peer_arg), 3)
self.assertEqual(peer_arg[0], SocketAddress('10.30.10.216', 40111))
self.assertEqual(
peer_arg[1],
SocketAddress('2001:db8:85a3:8d3:1319:8a2e:370:7348', 443))
self.assertEqual(peer_arg[2], SocketAddress('::ffff:0:0:0', 96))
def add_new_task(self, task):
if task.header.task_id in self.tasks:
raise RuntimeError("Task has been already added")
if not self.key_id:
raise ValueError("'key_id' is not set")
if not SocketAddress.is_proper_address(self.listen_address,
self.listen_port):
raise IOError("Incorrect socket address")
prev_pub_addr, prev_pub_port, prev_nat_type = self.node.pub_addr, self.node.pub_port, self.node.nat_type
self.node.pub_addr, self.node.pub_port, self.node.nat_type = yield self.get_external_address(
)
if prev_pub_addr != self.node.pub_addr or \
prev_pub_port != self.node.pub_port or \
prev_nat_type != self.node.nat_type:
self.update_task_signatures()
task.header.task_owner_address = self.listen_address
task.header.task_owner_port = self.listen_port
task.header.task_owner_key_id = self.key_id
task.header.task_owner = self.node
task.header.signature = self.sign_task_header(task.header)
self.dir_manager.clear_temporary(task.header.task_id,
undeletable=task.undeletable)
self.dir_manager.get_task_temporary_dir(task.header.task_id,
create=True)
task.register_listener(self)
task.task_status = TaskStatus.waiting
self.tasks[task.header.task_id] = task
ts = TaskState()
ts.status = TaskStatus.waiting
ts.outputs = task.get_output_names()
ts.total_subtasks = task.get_total_tasks()
ts.time_started = time.time()
self.tasks_states[task.header.task_id] = ts
if self.task_persistence:
self.dump_task(task.header.task_id)
logger.info("Task {} added".format(task.header.task_id))
self.notice_task_updated(task.header.task_id)
def test_pending_conn(self):
network = Network()
server = PendingConnectionsServer(None, network)
req_type = 0
final_failure_called = [False]
def final_failure(*args, **kwargs):
final_failure_called[0] = True
server.conn_established_for_type[req_type] = lambda x: x
server.conn_failure_for_type[req_type] = server.final_conn_failure
server.conn_final_failure_for_type[req_type] = final_failure
server._is_address_accessible = Mock(return_value=True)
server._add_pending_request(req_type,
self.node_info,
prv_port=self.node_info.prv_port,
pub_port=self.node_info.pub_port,
args={})
assert len(server.pending_connections) == 1
pending_conn = next(iter(list(server.pending_connections.values())))
final_failure_called[0] = False
server.final_conn_failure(pending_conn.id)
assert final_failure_called[0]
server.verified_conn(pending_conn.id)
assert len(server.pending_connections) == 0
final_failure_called[0] = False
server.final_conn_failure(pending_conn.id)
assert not final_failure_called[0]
server._add_pending_request(req_type,
self.node_info,
prv_port=self.node_info.prv_port,
pub_port=self.node_info.pub_port,
args={})
pending_conn = next(iter(list(server.pending_connections.values())))
server._mark_connected(pending_conn.id, "10.10.10.1", self.port)
assert pending_conn.status == PenConnStatus.Connected
assert SocketAddress("10.10.10.1",
self.port) == pending_conn.socket_addresses[0]
def dispatch(args):
if len(args) == 4 and args[1] == REQUESTING_NODE_KIND:
# I'm a requesting node,
# second arg is the data dir,
# third arg is the number of subtasks.
run_requesting_node(args[2], int(args[3]))
elif len(args) in [4, 5] and args[1] == COMPUTING_NODE_KIND:
# I'm a computing node,
# second arg is the data dir,
# third arg is the address to connect to,
# forth arg is the timeout (optional).
fail_after = float(args[4]) if len(args) == 5 else None
run_computing_node(args[2], SocketAddress.parse(args[3]),
fail_after=fail_after)
elif len(args) == 1:
# I'm the main script, run simulation
error_msg = run_simulation(num_computing_nodes=2, num_subtasks=4,
timeout=120)
if error_msg:
print("Dummy task computation failed:", error_msg)
sys.exit(1)
