def test_reuse_passive_live_macos_nok_nok(unused_tcp_port):
"""
Executive summary of this test
1. the original socket is obtained with will_close_then_reopen_socket=True
2. no activity is triggered on the socket
3. the socket is not closed but kept alive
4. port can not be re-bound in any mode
"""
custom_range = range(unused_tcp_port, unused_tcp_port + 1)
_, port, orig_sock = port_handler.find_available_port(
custom_range=custom_range, will_close_then_reopen_socket=True
)
assert port == unused_tcp_port
assert orig_sock is not None
assert orig_sock.fileno() != -1
# As long as the socket is kept alive this port can not be bound again...
with pytest.raises(port_handler.NoPortsInRangeException) as exc_info:
port_handler.find_available_port(custom_range=custom_range)
# ... not even when setting will_close_then_reopen_socket=True
with pytest.raises(port_handler.NoPortsInRangeException) as exc_info:
port_handler.find_available_port(
custom_range=custom_range, will_close_then_reopen_socket=True
)
def test_reuse_passive_live_linux_nok_ok(unused_tcp_port):
"""
Executive summary of this test
1. the original socket is obtained with will_close_then_reopen_socket=True
2. no activity is triggered on the socket
3. the socket is not closed but kept alive
4. port can not be re-bound in default mode...
5. ... but can with will_close_then_reopen_socket=True
"""
custom_range = range(unused_tcp_port, unused_tcp_port + 1)
# Opening original socket with will_close_then_reopen_socket=True
_, port, orig_sock = port_handler.find_available_port(
custom_range=custom_range, will_close_then_reopen_socket=True
)
assert port == unused_tcp_port
assert orig_sock is not None
assert orig_sock.fileno() != -1
# As long as the socket is kept alive this port can not be bound again...
with pytest.raises(port_handler.NoPortsInRangeException) as exc_info:
port_handler.find_available_port(custom_range=custom_range)
# ... but on Linux the port can be re-bound by setting this flag!
# This does not seem safe in a multi-user/-process environment!
_, port, sock = port_handler.find_available_port(
custom_range=custom_range, will_close_then_reopen_socket=True
)
assert port == unused_tcp_port
assert sock is not None
assert sock.fileno() != -1
def test_def_active_live_nok_nok(unused_tcp_port):
"""
Executive summary of this test
1. the original socket is obtained in default, recommended mode
2. activity is triggered on the socket using a dummy-server/client
3. socket is not closed but kept alive
4. port can not be re-bound in any mode while socket-object is live
"""
custom_range = range(unused_tcp_port, unused_tcp_port + 1)
host, port, orig_sock = port_handler.find_available_port(custom_range=custom_range)
assert port == unused_tcp_port
assert orig_sock is not None
assert orig_sock.fileno() != -1
# Now, run a dummy-server to actually use the socket a little, do NOT close socket
_simulate_server(host, port, orig_sock)
# Immediately trying to bind to the same port fails...
with pytest.raises(port_handler.NoPortsInRangeException) as exc_info:
host, port, sock = port_handler.find_available_port(custom_range=custom_range)
# ... also using will_close_then_reopen_socket=True
with pytest.raises(port_handler.NoPortsInRangeException) as exc_info:
host, port, sock = port_handler.find_available_port(
custom_range=custom_range, will_close_then_reopen_socket=True
)
def test_def_active_close_linux_nok_nok(unused_tcp_port):
"""
Executive summary of this test
1. the original socket is obtained in default, recommended mode
2. activity is triggered on the socket using a dummy-server/client
3. socket is closed
4. after socket is closed the port can not be re-bound in default mode (TIME_WAIT?)...
5. ...nor with will_close_then_reopen_socket=True (not ignoring TIME_WAIT?)
"""
custom_range = range(unused_tcp_port, unused_tcp_port + 1)
host, port, orig_sock = port_handler.find_available_port(custom_range=custom_range)
assert port == unused_tcp_port
assert orig_sock is not None
assert orig_sock.fileno() != -1
# Now, run a dummy-server to actually use the socket a little, then close it
_simulate_server(host, port, orig_sock)
orig_sock.close()
# Immediately trying to bind to the same port fails
with pytest.raises(port_handler.NoPortsInRangeException) as exc_info:
host, port, sock = port_handler.find_available_port(custom_range=custom_range)
# On Linux, setting will_close_then_reopen_socket=True in subsequent calls do
# NOT allow reusing the port in this case
with pytest.raises(port_handler.NoPortsInRangeException) as exc_info:
host, port, sock = port_handler.find_available_port(
custom_range=custom_range, will_close_then_reopen_socket=True
)
def test_reuse_active_close_nok_ok(unused_tcp_port):
"""
Executive summary of this test
1. the original socket is obtained with will_close_then_reopen_socket=True
2. activity is triggered on the socket using a dummy-server/client
3. socket is closed
4. port can not be re-bound in default mode (TIME_WAIT?)...
5. ... but can with will_close_then_reopen_socket=True (ignoring TIME_WAIT)
"""
custom_range = range(unused_tcp_port, unused_tcp_port + 1)
# Note: Setting will_close_then_reopen_socket=True on original socket
host, port, orig_sock = port_handler.find_available_port(
custom_range=custom_range, will_close_then_reopen_socket=True
)
assert port == unused_tcp_port
assert orig_sock is not None
assert orig_sock.fileno() != -1
# Run a dummy-server to actually use the socket a little, then close it
_simulate_server(host, port, orig_sock)
orig_sock.close()
# Using will_close_then_reopen_socket=False fails...
with pytest.raises(port_handler.NoPortsInRangeException) as exc_info:
port_handler.find_available_port(custom_range=custom_range)
# ... but using will_close_then_reopen_socket=True succeeds
_, port, sock = port_handler.find_available_port(
custom_range=custom_range, will_close_then_reopen_socket=True
)
assert port == unused_tcp_port
assert sock is not None
assert sock.fileno() != -1
def test_reuse_active_live_nok_nok(unused_tcp_port):
"""
Executive summary of this test
1. the original socket is obtained with will_close_then_reopen_socket=True
2. activity is triggered on the socket using a dummy-server/client
3. socket is not closed but kept alive
4. port can not be re-bound in default mode (TIME_WAIT?)...
5. ... but can with will_close_then_reopen_socket=True (ignoring TIME_WAIT)
"""
custom_range = range(unused_tcp_port, unused_tcp_port + 1)
host, port, orig_sock = port_handler.find_available_port(
custom_range=custom_range, will_close_then_reopen_socket=True
)
assert port == unused_tcp_port
assert orig_sock is not None
assert orig_sock.fileno() != -1
# Run a dummy-server to actually use the socket a little, then close it
_simulate_server(host, port, orig_sock)
# Even with "will_close_then_reopen_socket"=True when obtaining original
# socket, subsequent calls fails
with pytest.raises(port_handler.NoPortsInRangeException) as exc_info:
port_handler.find_available_port(custom_range=custom_range)
with pytest.raises(port_handler.NoPortsInRangeException) as exc_info:
_, port, sock = port_handler.find_available_port(
custom_range=custom_range, will_close_then_reopen_socket=True
)
def test_invalid_host_name():
invalid_host = "invalid_host"
with pytest.raises(port_handler.InvalidHostException) as exc_info:
port_handler.find_available_port(custom_host=invalid_host)
assert ("Trying to bind socket with what looks "
f"like an invalid hostname ({invalid_host})") in str(
exc_info.value)
def test_find_available_port_forced(unused_tcp_port):
custom_range = range(unused_tcp_port, unused_tcp_port)
host, port, sock = port_handler.find_available_port(
custom_range=custom_range, custom_host="127.0.0.1")
assert port == unused_tcp_port
assert sock is not None
assert sock.fileno() != -1
def test_find_available_port(unused_tcp_port):
custom_range = range(unused_tcp_port, unused_tcp_port + 1)
host, port, sock = port_handler.find_available_port(custom_range=custom_range)
assert host is not None
assert port is not None
assert port in custom_range
assert sock is not None
assert sock.fileno() != -1
def run_server(args=None, debug=False):
if args is None:
args = parse_args()
if "ERT_STORAGE_TOKEN" in os.environ:
authtoken = os.environ["ERT_STORAGE_TOKEN"]
else:
authtoken = generate_authtoken()
os.environ["ERT_STORAGE_TOKEN"] = authtoken
lockfile = Path.cwd() / "storage_server.json"
if lockfile.exists():
sys.exit("'storage_server.json' already exists")
config_args = {}
if args.debug or debug:
config_args.update(reload=True,
reload_dirs=[os.path.dirname(ert_shared_path)])
os.environ["ERT_STORAGE_DEBUG"] = "1"
_, _, sock = port_handler.find_available_port(custom_host=args.host)
connection_info = _create_connection_info(sock, authtoken)
# Appropriated from uvicorn.main:run
os.environ["ERT_STORAGE_NO_TOKEN"] = "1"
if args.enable_new_storage:
args.database_url = "sqlite:///ert.db"
if args.database_url:
os.environ["ERT_STORAGE_DATABASE_URL"] = args.database_url
config = uvicorn.Config("ert_storage.app:app", **config_args)
else:
# Dark Storage imports from ERT Storage, which connects to the database
# at startup. We set the database URL to an SQLite in-memory database so
# that the import succeeds.
os.environ["ERT_STORAGE_DATABASE_URL"] = "sqlite://"
os.environ["ERT_STORAGE_RES_CONFIG"] = args.config or find_ert_config()
config = uvicorn.Config("ert_shared.dark_storage.app:app",
**config_args)
server = Server(config, json.dumps(connection_info), lockfile)
print("Storage server is ready to accept requests. Listening on:")
for url in connection_info["urls"]:
print(f" {url}")
print(f"\nOpenAPI Docs: {url}/docs", file=sys.stderr)
if args.debug or debug:
print("\tRunning in NON-SECURE debug mode.\n")
os.environ["ERT_STORAGE_NO_TOKEN"] = "1"
else:
print("\tUsername: __token__")
print(f"\tPassword: {connection_info['authtoken']}\n")
if config.should_reload:
supervisor = ChangeReload(config, target=server.run, sockets=[sock])
supervisor.run()
else:
server.run(sockets=[sock])
def test_reuse_passive_close_ok_ok(unused_tcp_port):
"""
Executive summary of this test
1. the original socket is obtained with will_close_then_reopen_socket=True
2. no activity is triggered on the socket
3. the socket is closed
4. port can be re-bound in any mode
"""
custom_range = range(unused_tcp_port, unused_tcp_port + 1)
_, port, orig_sock = port_handler.find_available_port(
custom_range=custom_range,
custom_host="127.0.0.1",
will_close_then_reopen_socket=True,
)
assert port == unused_tcp_port
assert orig_sock is not None
assert orig_sock.fileno() != -1
orig_sock.close()
# When we close the socket without actually having used it, it is
# immediately reusable with or without setting will_close_then_reopen_socket
_, port, sock = port_handler.find_available_port(custom_range=custom_range,
custom_host="127.0.0.1")
assert port == unused_tcp_port
assert sock is not None
assert sock.fileno() != -1
# we want to try again, so close it
sock.close()
_, port, sock = port_handler.find_available_port(
custom_range=custom_range,
custom_host="127.0.0.1",
will_close_then_reopen_socket=True,
)
assert port == unused_tcp_port
assert sock is not None
assert sock.fileno() != -1
def test_gc_closes_socket(unused_tcp_port):
custom_range = range(unused_tcp_port, unused_tcp_port + 1)
_, port, orig_sock = port_handler.find_available_port(
custom_range=custom_range, custom_host="127.0.0.1")
assert port == unused_tcp_port
assert orig_sock is not None
assert orig_sock.fileno() != -1
with pytest.raises(port_handler.NoPortsInRangeException):
port_handler.find_available_port(custom_range=custom_range,
custom_host="127.0.0.1")
with pytest.raises(port_handler.NoPortsInRangeException):
port_handler.find_available_port(
custom_range=custom_range,
will_close_then_reopen_socket=True,
custom_host="127.0.0.1",
)
orig_sock = None
_, port, orig_sock = port_handler.find_available_port(
custom_range=custom_range, custom_host="127.0.0.1")
assert port == unused_tcp_port
assert orig_sock is not None
assert orig_sock.fileno() != -1
def test_def_passive_live_nok_nok_close_ok_ok(unused_tcp_port):
"""
Executive summary of this test
1. the original socket is obtained in default, recommended mode
2. no activity is triggered on the socket
3. port is not closed but kept alive
4. port can not be re-bound in any mode while socket-object is live
5. port is closed
6. after socket is closed the port can immediately be re-bound in any mode
"""
custom_range = range(unused_tcp_port, unused_tcp_port + 1)
# Opening original socket with will_close_then_reopen_socket=False
_, port, orig_sock = port_handler.find_available_port(
custom_range=custom_range, custom_host="127.0.0.1")
assert port == unused_tcp_port
assert orig_sock is not None
assert orig_sock.fileno() != -1
# When the socket is kept open, this port can not be reused
# with or without setting will_close_then_reopen_socket
with pytest.raises(port_handler.NoPortsInRangeException):
port_handler.find_available_port(custom_range=custom_range,
custom_host="127.0.0.1")
with pytest.raises(port_handler.NoPortsInRangeException):
port_handler.find_available_port(
custom_range=custom_range,
custom_host="127.0.0.1",
will_close_then_reopen_socket=True,
)
orig_sock.close()
# When we close the socket without actually having used it, it is
# immediately reusable with or without setting will_close_then_reopen_socket
_, port, sock = port_handler.find_available_port(custom_range=custom_range,
custom_host="127.0.0.1")
assert port == unused_tcp_port
assert sock is not None
assert sock.fileno() != -1
# we want to try again, so close it
sock.close()
_, port, sock = port_handler.find_available_port(
custom_range=custom_range,
custom_host="127.0.0.1",
will_close_then_reopen_socket=True,
)
assert port == unused_tcp_port
assert sock is not None
assert sock.fileno() != -1
def test_create_connection_string(monkeypatch):
authtoken = "very_secret_token"
_, _, sock = port_handler.find_available_port()
_storage_main._create_connection_info(sock, authtoken)
assert "ERT_STORAGE_CONNECTION_STRING" in os.environ
connection_string = json.loads(os.environ["ERT_STORAGE_CONNECTION_STRING"])
assert "urls" in connection_string
assert "authtoken" in connection_string
assert len(connection_string["urls"]) == 3
del os.environ["ERT_STORAGE_CONNECTION_STRING"]
def _get_executor(custom_port_range, name="local"):
# See https://github.com/equinor/ert/pull/2757#discussion_r794368854
_, port, sock = find_available_port(custom_range=custom_port_range)
sock.close() # do this explicitly, not relying on GC
if name == "local":
cluster_kwargs = {
"silence_logs": "debug",
}
return LocalDaskExecutor(**cluster_kwargs)
elif name == "lsf":
LSFJob._submit_job = _eq_submit_job
cluster_kwargs = {
"queue": "mr",
"project": None,
"cores": 1,
"memory": "1GB",
"use_stdin": True,
"n_workers": 2,
"silence_logs": "debug",
"scheduler_options": {
"port": port
},
}
return DaskExecutor(
cluster_class="dask_jobqueue.LSFCluster",
cluster_kwargs=cluster_kwargs,
debug=True,
)
elif name == "pbs":
cluster_kwargs = {
"n_workers": 10,
"queue": "normal",
"project": "ERT-TEST",
"local_directory": "$TMPDIR",
"cores": 1,
"memory": "32gb",
"resource_spec": "select=1:ncpus=1:mem=32gb",
"scheduler_options": {
"port": port
},
"extra": ["--worker-port", "51820:51840"],
}
return DaskExecutor(
cluster_class="dask_jobqueue.PBSCluster",
cluster_kwargs=cluster_kwargs,
debug=True,
)
else:
raise ValueError(f"Unknown executor name {name}")
def __init__(
self,
custom_port_range: range = None,
use_token: bool = True,
generate_cert: bool = True,
) -> None:
self.host, self.port, self._socket_handle = port_handler.find_available_port(
custom_range=custom_port_range)
self.protocol = "wss" if generate_cert else "ws"
self.url = f"{self.protocol}://{self.host}:{self.port}"
self.client_uri = f"{self.url}/client"
self.dispatch_uri = f"{self.url}/dispatch"
if generate_cert:
cert, key, pw = _generate_certificate(ip_address=self.host)
else:
cert, key, pw = None, None, None # type: ignore
self.cert = cert
self._key = key
self._key_pw = pw
self.token = _generate_authentication() if use_token else None
