def get_socket_addresses(self, node_info, prv_port=None, pub_port=None): """ Change node info into tcp addresses. Adds a suggested address. :param Node node_info: node information :param prv_port: private port that should be used :param pub_port: public port that should be used :return: """ prv_port = prv_port or node_info.prv_port pub_port = pub_port or node_info.pub_port socket_addresses = super().get_socket_addresses(node_info=node_info, prv_port=prv_port, pub_port=pub_port) address = self.client.get_suggested_addr(node_info.key) if not address: return socket_addresses if self._is_address_valid(address, prv_port): socket_address = SocketAddress(address, prv_port) self._prepend_address(socket_addresses, socket_address) if self._is_address_valid(address, pub_port): socket_address = SocketAddress(address, pub_port) self._prepend_address(socket_addresses, socket_address) return socket_addresses[:MAX_CONNECT_SOCKET_ADDRESSES]
def test_get_socket_addresses(self): key_id = 'abcd' address = '10.0.0.1' prv_port = 1 pub_port = 2 node_info = mock.Mock(key=key_id) self.service.suggested_address[key_id] = address result = self.service.get_socket_addresses(node_info, prv_port, pub_port) assert SocketAddress(address, prv_port) in result assert SocketAddress(address, pub_port) in result
def __expect_exception(self, value, exception): try: SocketAddress.parse(value) suffix = ', not succeed' except exception: return except: exc = sys.exc_info()[0] suffix = ', not ' + exc.__name__ self.fail('SocketAddress.parse("' + str(value) + '") should raise ' + exception.__name__ + suffix)
def parse_http_addr(ctx, param, value): del ctx, param if value: try: http_prefix = 'https://' if not value.startswith(http_prefix): raise click.BadParameter("Address without https:// prefix" "specified: {}".format(value)) SocketAddress.parse(value[len(http_prefix):]) return value except ipaddress.AddressValueError as e: raise click.BadParameter( "Invalid network address specified: {}".format(e)) return None
def test_get_socket_addresses(self): server = PendingConnectionsServer(None, Network()) node = self.node_info res = server.get_socket_addresses(node, prv_port=node.prv_port, pub_port=node.pub_port) self.assertEqual(res, [ SocketAddress(self.node_info.pub_addr, self.node_info.pub_port), SocketAddress(self.node_info.prv_addresses[0], self.node_info.prv_port) ]) node.pub_addr = "10.10.10.10" res = server.get_socket_addresses(node, prv_port=node.prv_port, pub_port=node.pub_port) self.assertEqual(len(res), 2) self.assertEqual(res[0].address, node.pub_addr) self.assertEqual(res[0].port, node.pub_port) node.pub_port = 1023 res = server.get_socket_addresses(node, prv_port=node.prv_port, pub_port=node.pub_port) self.assertEqual(len(res), 2) self.assertEqual(res[0].address, node.pub_addr) self.assertEqual(res[0].port, 1023) node.prv_addresses = [ "10.10.10.1", "10.10.10.2", "10.10.10.3", "10.10.10.4" ] res = server.get_socket_addresses(node, prv_port=node.prv_port, pub_port=node.pub_port) self.assertEqual(len(res), 5) self.assertEqual(res[4].address, node.prv_addresses[-1]) self.assertEqual(res[4].port, 1233) for i in range(4): self.assertEqual(res[i + 1].address, node.prv_addresses[i]) self.assertEqual(res[i + 1].port, node.prv_port) node.pub_addr = None res = server.get_socket_addresses(node, prv_port=node.prv_port, pub_port=node.pub_port) self.assertEqual(len(res), 4) for i in range(4): self.assertEqual(res[i].address, node.prv_addresses[i]) self.assertEqual(res[i].port, node.prv_port)
def connect(self, ip, port_): try: sa = SocketAddress(ip, int(port_)) Network.client.connect((sa.address, sa.port)) except Exception as exc: return CommandResult( error="Cannot connect to {}:{}: {}".format(ip, port_, exc))
def test_refresh_peers(self): sa = SocketAddress('127.0.0.1', 11111) node = mock.MagicMock() node.key = encode_hex(urandom(64))[2:] node.key_id = node.key node.address = sa node2 = mock.MagicMock() node2.key = encode_hex(urandom(64))[2:] node2.key_id = node2.key node2.address = sa self.service.add_peer(node) self.service.add_peer(node2) self.service.peers[node.key].last_message_time = time.time() + 1000 self.service.peers[node2.key].last_message_time = time.time() + 1000 self.service.config_desc.opt_peer_num = 1000 assert len(self.service.peers) == 2 self.service.sync_network() assert len(self.service.peers) == 2 self.service.last_refresh_peers = 0 self.service.last_peers_request = 0 self.service._peer_dbg_time_threshold = 0 self.service.sync_network() # disabled assert len(self.service.peers) == 2
def test_run_computing_node(self, mock_reactor): client = runner.run_computing_node(self.path, SocketAddress("127.0.0.1", 40102)) environments = list(client.environments_manager.environments) self.assertTrue(any(env.get_id() == task.DummyTask.ENVIRONMENT_NAME for env in environments)) client.quit()
def test_refresh_peers(self): sa = SocketAddress('127.0.0.1', 11111) node = MagicMock() node.key = EllipticalKeysAuth(self.path, "TESTPRIV", "TESTPUB").get_key_id() node.key_id = node.key node.address = sa node2 = MagicMock() node2.key = EllipticalKeysAuth(self.path, "TESTPRIV2", "TESTPUB2").get_key_id() node2.key_id = node2.key node2.address = sa self.service.add_peer(node.key, node) self.service.add_peer(node2.key, node2) self.service.peers[node.key].last_message_time = time.time() + 1000 self.service.peers[node2.key].last_message_time = time.time() + 1000 self.service.config_desc.opt_peer_num = 1000 assert len(self.service.peers) == 2 self.service.sync_network() assert len(self.service.peers) == 2 self.service.last_refresh_peers = 0 self.service.last_peers_request = 0 self.service._peer_dbg_time_threshold = 0 self.service.sync_network() # disabled assert len(self.service.peers) == 2
def test_many_peers(self, config_logging, mock_node): addr1 = '10.30.10.216:40111' addr2 = '10.30.10.214:3333' runner = CliRunner() args = self.args + ['--peer', addr1, '--peer', addr2] return_value = runner.invoke(start, args, catch_exceptions=False) self.assertEqual(return_value.exit_code, 0) mock_node.assert_has_calls( [call().run(use_rpc=True), call().add_tasks([])], any_order=True) call_names = [name for name, arg, kwarg in mock_node.mock_calls] self.assertTrue('().connect_with_peers' in call_names) peer_num = call_names.index('().connect_with_peers') peer_arg = mock_node.mock_calls[peer_num][1][0] self.assertEqual(len(peer_arg), 2) self.assertEqual(peer_arg[0], SocketAddress.parse(addr1)) self.assertEqual(peer_arg[1], SocketAddress.parse(addr2))
def test_run_computing_node(self, mock_config_logging, mock_reactor, _): client = runner.run_computing_node(self.path, SocketAddress("127.0.0.1", 40102)) assert task.DummyTask.ENVIRONMENT_NAME in \ client.environments_manager.environments self.assertTrue(mock_reactor.run.called) self.assertTrue(mock_config_logging.called) client.quit()
def connect(self, socket_address): if isinstance(socket_address, Iterable): socket_address = SocketAddress(socket_address[0], int(socket_address[1])) log.debug("P2pservice connecting to %s on port %s", socket_address.address, socket_address.port) self.p2pservice.connect(socket_address)
def parse_rpc_address(ctx, param, value): del ctx, param if value: try: return SocketAddress.parse(value) except AddressValueError as e: raise click.BadParameter( "Invalid RPC address specified: {}".format(e.message))
def __expect_valid(self, value): if value[0] == '[': addr, port = value.split(']:') addr = addr[1:] else: addr, port = value.split(':') a = SocketAddress.parse(value) self.assertTrue(a.address == addr) self.assertTrue(a.port == int(port))
def traverse_nat(self, key_id, addr, port, conn_id, super_key_id): connect_info = TCPConnectInfo( [SocketAddress(addr, port)], self.__connection_for_traverse_nat_established, self.__connection_for_traverse_nat_failure) self.network.connect(connect_info, client_key_id=key_id, conn_id=conn_id, super_key_id=super_key_id)
def get_socket_addresses(self, node_info, port, key_id): if self.client.get_suggested_conn_reverse(key_id): return [] socket_addresses = PendingConnectionsServer.get_socket_addresses( self, node_info, port, key_id) addr = self.client.get_suggested_addr(key_id) if addr: socket_addresses = [SocketAddress(addr, port)] + socket_addresses return socket_addresses
def parse_peer(ctx, param, value): del ctx, param addresses = [] for arg in value: try: addresses.append(SocketAddress.parse(arg)) except AddressValueError as e: raise click.BadParameter( "Invalid peer address specified: {}".format(e.message)) return addresses
def test_address_accessible(self): config = Mock() config.use_ipv6 = False server = PendingConnectionsServer(config, Mock()) assert not server._is_address_accessible(None) sockv4 = SocketAddress('8.8.8.8', 40100) sockv6 = SocketAddress('2001:0db8:85a3:0000:0000:8a2e:abcd:efea', 40100) assert server._is_address_accessible(sockv4) assert not server._is_address_accessible(sockv6) server.use_ipv6 = True assert server._is_address_accessible(sockv4) assert server._is_address_accessible(sockv6)
def parse_node_addr(ctx, param, value): del ctx, param if value: try: SocketAddress(value, 1) return value except AddressValueError as e: raise click.BadParameter( "Invalid network address specified: {}".format(e.message)) return ''
def parse_rpc_address(ctx, param, value): del ctx, param value = to_unicode(value) if value: try: return SocketAddress.parse(value) except ipaddress.AddressValueError as e: raise click.BadParameter( "Invalid RPC address specified: {}".format(e)) return None
def test_zone_index(self): base_address = u"fe80::3" address = u"fe80::3%eth0" port = 1111 sa = SocketAddress(address, port) assert sa.address == base_address assert sa.port == port address = u"fe80::3%1" sa = SocketAddress(address, port) assert sa.address == base_address address = u"fe80::3%en0" sa = SocketAddress(address, port) assert sa.address == base_address address = base_address sa = SocketAddress(address, port) assert sa.address == base_address
def check_rpc_address(ctx, param, address): split = address.rsplit(':', 1) host, port = split[0], int(split[1]) try: SocketAddress(host, port) except AddressValueError as e: return click.BadParameter( "Invalid network address specified: {}".format(e.message)) return WebSocketAddress(host, port, u'golem')
def test_connect_success(self, connection_established, createSocket): createSocket.return_value = socket = mock.Mock() socket.fileno = mock.Mock(return_value=0) socket.getsockopt = mock.Mock(return_value=None) socket.connect_ex = mock.Mock(return_value=EISCONN) addr = SocketAddress('127.0.0.1', 40102) self.service.connect(addr) time.sleep(0.1) assert connection_established.called assert connection_established.call_args[0][0] is not None assert connection_established.call_args[1]['conn_id'] is not None
def test_runner_dispatch_computing( self, mock_run_simulation, mock_run_computing_node, mock_run_requesting_node): args = ["runner.py", runner.COMPUTING_NODE_KIND, self.path, "1.2.3.4:5678"] runner.dispatch(args) self.assertFalse(mock_run_requesting_node.called) self.assertTrue(mock_run_computing_node.called) self.assertEqual(mock_run_computing_node.call_args[0], (self.path, SocketAddress("1.2.3.4", 5678))) self.assertEqual(mock_run_computing_node.call_args[1], {"fail_after": None}) self.assertFalse(mock_run_simulation.called)
def test_runner_dispatch_computing_with_failure( self, mock_run_simulation, mock_run_computing_node, mock_run_requesting_node): args = ["runner.py", runner.COMPUTING_NODE_KIND, self.path, "10.0.255.127:16000", "25"] runner.dispatch(args) self.assertFalse(mock_run_requesting_node.called) self.assertTrue(mock_run_computing_node.called) self.assertEqual(mock_run_computing_node.call_args[0], (self.path, SocketAddress("10.0.255.127", 16000))) self.assertEqual(mock_run_computing_node.call_args[1], {"fail_after": 25.0}) self.assertFalse(mock_run_simulation.called)
def __str__(self): ip4, ip6 = 'ip4', 'ip6' proto = self.proto or 'tcp' port = self.port or IPFS_DEFAULT_TCP_PORT sa = SocketAddress(self.ip_address, port) if self.encap_proto: return self.pattern_encap.format(ip6 if sa.ipv6 else ip4, self.ip_address, proto, port, self.encap_proto, self.node_id) else: return self.pattern.format(ip6 if sa.ipv6 else ip4, self.ip_address, proto, port, self.node_id)
def test_peers(self, config_logging, mock_node): runner = CliRunner() return_value = runner.invoke( start, self.args + [ '--peer', u'10.30.10.216:40111', u'--peer', u'[2001:db8:85a3:8d3:1319:8a2e:370:7348]:443', '--peer', '[::ffff:0:0:0]:96' ], catch_exceptions=False) self.assertEqual(return_value.exit_code, 0) mock_node.assert_has_calls( [call().run(use_rpc=True), call().add_tasks([])], any_order=True) call_names = [name for name, arg, kwarg in mock_node.mock_calls] self.assertTrue('().connect_with_peers' in call_names) peer_num = call_names.index('().connect_with_peers') peer_arg = mock_node.mock_calls[peer_num][1][0] self.assertEqual(len(peer_arg), 3) self.assertEqual(peer_arg[0], SocketAddress('10.30.10.216', 40111)) self.assertEqual( peer_arg[1], SocketAddress('2001:db8:85a3:8d3:1319:8a2e:370:7348', 443)) self.assertEqual(peer_arg[2], SocketAddress('::ffff:0:0:0', 96))
def add_new_task(self, task): if task.header.task_id in self.tasks: raise RuntimeError("Task has been already added") if not self.key_id: raise ValueError("'key_id' is not set") if not SocketAddress.is_proper_address(self.listen_address, self.listen_port): raise IOError("Incorrect socket address") prev_pub_addr, prev_pub_port, prev_nat_type = self.node.pub_addr, self.node.pub_port, self.node.nat_type self.node.pub_addr, self.node.pub_port, self.node.nat_type = yield self.get_external_address( ) if prev_pub_addr != self.node.pub_addr or \ prev_pub_port != self.node.pub_port or \ prev_nat_type != self.node.nat_type: self.update_task_signatures() task.header.task_owner_address = self.listen_address task.header.task_owner_port = self.listen_port task.header.task_owner_key_id = self.key_id task.header.task_owner = self.node task.header.signature = self.sign_task_header(task.header) self.dir_manager.clear_temporary(task.header.task_id, undeletable=task.undeletable) self.dir_manager.get_task_temporary_dir(task.header.task_id, create=True) task.register_listener(self) task.task_status = TaskStatus.waiting self.tasks[task.header.task_id] = task ts = TaskState() ts.status = TaskStatus.waiting ts.outputs = task.get_output_names() ts.total_subtasks = task.get_total_tasks() ts.time_started = time.time() self.tasks_states[task.header.task_id] = ts if self.task_persistence: self.dump_task(task.header.task_id) logger.info("Task {} added".format(task.header.task_id)) self.notice_task_updated(task.header.task_id)
def test_pending_conn(self): network = Network() server = PendingConnectionsServer(None, network) req_type = 0 final_failure_called = [False] def final_failure(*args, **kwargs): final_failure_called[0] = True server.conn_established_for_type[req_type] = lambda x: x server.conn_failure_for_type[req_type] = server.final_conn_failure server.conn_final_failure_for_type[req_type] = final_failure server._is_address_accessible = Mock(return_value=True) server._add_pending_request(req_type, self.node_info, prv_port=self.node_info.prv_port, pub_port=self.node_info.pub_port, args={}) assert len(server.pending_connections) == 1 pending_conn = next(iter(list(server.pending_connections.values()))) final_failure_called[0] = False server.final_conn_failure(pending_conn.id) assert final_failure_called[0] server.verified_conn(pending_conn.id) assert len(server.pending_connections) == 0 final_failure_called[0] = False server.final_conn_failure(pending_conn.id) assert not final_failure_called[0] server._add_pending_request(req_type, self.node_info, prv_port=self.node_info.prv_port, pub_port=self.node_info.pub_port, args={}) pending_conn = next(iter(list(server.pending_connections.values()))) server._mark_connected(pending_conn.id, "10.10.10.1", self.port) assert pending_conn.status == PenConnStatus.Connected assert SocketAddress("10.10.10.1", self.port) == pending_conn.socket_addresses[0]
def dispatch(args): if len(args) == 4 and args[1] == REQUESTING_NODE_KIND: # I'm a requesting node, # second arg is the data dir, # third arg is the number of subtasks. run_requesting_node(args[2], int(args[3])) elif len(args) in [4, 5] and args[1] == COMPUTING_NODE_KIND: # I'm a computing node, # second arg is the data dir, # third arg is the address to connect to, # forth arg is the timeout (optional). fail_after = float(args[4]) if len(args) == 5 else None run_computing_node(args[2], SocketAddress.parse(args[3]), fail_after=fail_after) elif len(args) == 1: # I'm the main script, run simulation error_msg = run_simulation(num_computing_nodes=2, num_subtasks=4, timeout=120) if error_msg: print("Dummy task computation failed:", error_msg) sys.exit(1)