def test_grpc_watch_thread_server_complex_cycle_2():
# Server goes down, comes back up as the same server three times, then goes away and comes
# back as a new server
port = find_free_port()
fixed_server_id = "fixed_id"
events = []
called = {}
def on_disconnect():
events.append("on_disconnect")
def on_reconnected():
events.append("on_reconnected")
def on_updated(_):
events.append("on_updated")
def on_error():
called["on_error"] = True
events.append("on_error")
# Create initial server
open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id)
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1 # This is a faster watch interval than we would use in practice
shutdown_event, watch_thread = create_grpc_watch_thread(
client,
on_disconnect=on_disconnect,
on_reconnected=on_reconnected,
on_updated=on_updated,
on_error=on_error,
watch_interval=watch_interval,
max_reconnect_attempts=3,
)
watch_thread.start()
time.sleep(watch_interval * 3)
cycles = 3
for x in range(1, cycles + 1):
# Simulate server restart three times with same server ID
client.shutdown_server()
wait_for_condition(lambda: events.count("on_disconnect") == x, watch_interval)
open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id)
wait_for_condition(lambda: events.count("on_reconnected") == x, watch_interval)
# Simulate server failure
client.shutdown_server()
# Wait for reconnect attempts to exhaust and on_error callback to be called
wait_for_condition(lambda: called.get("on_error"), watch_interval)
shutdown_event.set()
watch_thread.join()
assert events[-1] == "on_error"
def test_grpc_watch_thread_server_update():
port = find_free_port()
called = {}
def on_updated():
called["yup"] = True
# Create initial server
server_process = open_server_process(port=port, socket=None)
try:
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 4
shutdown_event, watch_thread = create_grpc_watch_thread(
client, on_updated=on_updated, watch_interval=watch_interval)
watch_thread.start()
time.sleep(watch_interval * 2)
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
assert not called
# Create updated server
server_process = open_server_process(port=port, socket=None)
try:
time.sleep(watch_interval * 2)
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
shutdown_event.set()
watch_thread.join()
assert called
def test_grpc_watch_thread_server_reconnect():
port = find_free_port()
fixed_server_id = "fixed_id"
called = {}
def on_disconnect(location_name):
assert location_name == "test_location"
called["on_disconnect"] = True
def on_reconnected(location_name):
assert location_name == "test_location"
called["on_reconnected"] = True
def should_not_be_called(*args, **kwargs):
raise Exception("This method should not be called")
# Create initial server
server_process = open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1
shutdown_event, watch_thread = create_grpc_watch_thread(
"test_location",
client,
on_disconnect=on_disconnect,
on_reconnected=on_reconnected,
on_updated=should_not_be_called,
on_error=should_not_be_called,
watch_interval=watch_interval,
)
watch_thread.start()
time.sleep(watch_interval * 3)
# Wait three seconds, simulate restart server, wait three seconds
interrupt_ipc_subprocess_pid(server_process.pid)
wait_for_condition(lambda: called.get("on_disconnect"), watch_interval)
server_process = open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
wait_for_condition(lambda: called.get("on_reconnected"), watch_interval)
shutdown_event.set()
watch_thread.join()
def test_server_socket():
with safe_tempfile_path() as skt:
server_process = open_server_process(port=None, socket=skt)
try:
assert DagsterGrpcClient(socket=skt).ping("foobar") == "foobar"
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
def test_server_port():
port = find_free_port()
server_process = open_server_process(port=port, socket=None)
assert server_process is not None
try:
assert DagsterGrpcClient(port=port).ping("foobar") == "foobar"
finally:
if server_process is not None:
interrupt_ipc_subprocess_pid(server_process.pid)
def test_fixed_server_id():
port = find_free_port()
server_process = open_server_process(port=port,
socket=None,
fixed_server_id="fixed_id")
assert server_process is not None
try:
api_client = DagsterGrpcClient(port=port)
assert api_client.get_server_id() == "fixed_id"
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
def test_grpc_watch_thread_server_update():
port = find_free_port()
called = {}
def on_updated(location_name, _):
assert location_name == "test_location"
called["yup"] = True
# Create initial server
server_process = open_server_process(port=port, socket=None)
try:
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1
shutdown_event, watch_thread = create_grpc_watch_thread(
"test_location",
client,
on_updated=on_updated,
watch_interval=watch_interval,
)
watch_thread.start()
time.sleep(watch_interval * 3)
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
assert not called
# Create updated server
server_process = open_server_process(port=port, socket=None)
try:
wait_for_condition(lambda: called, interval=watch_interval)
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
shutdown_event.set()
watch_thread.join()
assert called
def test_grpc_watch_thread_server_error():
port = find_free_port()
fixed_server_id = "fixed_id"
called = {}
def on_disconnect():
called["on_disconnect"] = True
def on_error():
called["on_error"] = True
def should_not_be_called():
raise Exception("This method should not be called")
# Create initial server
server_process = open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1
max_reconnect_attempts = 3
shutdown_event, watch_thread = create_grpc_watch_thread(
client,
on_disconnect=on_disconnect,
on_reconnected=should_not_be_called,
on_updated=should_not_be_called,
on_error=on_error,
watch_interval=watch_interval,
max_reconnect_attempts=max_reconnect_attempts,
)
watch_thread.start()
# Wait three seconds, simulate restart failure
time.sleep(watch_interval * 3)
interrupt_ipc_subprocess_pid(server_process.pid)
# Wait for reconnect attempts to exhaust and on_error callback to be called
start_time = time.time()
while not called.get("on_error"):
if time.time() - start_time > 30:
break
time.sleep(1)
shutdown_event.set()
watch_thread.join()
assert called["on_disconnect"]
assert called["on_error"]
def test_empty_executable_args():
port = find_free_port()
python_file = file_relative_path(__file__, "grpc_repo.py")
loadable_target_origin = LoadableTargetOrigin(executable_path="",
python_file=python_file)
# with an empty executable_path, the args change
process = None
try:
process = open_server_process(
port, socket=None, loadable_target_origin=loadable_target_origin)
assert process.args[:3] == ["dagster", "api", "grpc"]
finally:
if process:
process.terminate()
def test_python_environment_args():
port = find_free_port()
python_file = file_relative_path(__file__, "grpc_repo.py")
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable, python_file=python_file)
process = None
try:
process = open_server_process(
port, socket=None, loadable_target_origin=loadable_target_origin)
assert process.args[:5] == [
sys.executable, "-m", "dagster", "api", "grpc"
]
finally:
if process:
process.terminate()
def create_server_process():
port = find_free_port()
server_process = open_server_process(port=port, socket=None)
assert server_process is not None
return port, server_process
def test_grpc_watch_thread_server_complex_cycle():
# Server goes down, comes back up as the same server three times, then goes away and comes
# back as a new server
port = find_free_port()
fixed_server_id = "fixed_id"
events = []
def on_disconnect():
events.append("on_disconnect")
def on_reconnected():
events.append("on_reconnected")
def on_updated():
events.append("on_updated")
def on_error():
events.append("on_error")
# Create initial server
open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1 # This is a faster watch interval than we would use in practice
shutdown_event, watch_thread = create_grpc_watch_thread(
client,
on_disconnect=on_disconnect,
on_reconnected=on_reconnected,
on_updated=on_updated,
on_error=on_error,
watch_interval=watch_interval,
max_reconnect_attempts=5,
)
watch_thread.start()
time.sleep(watch_interval * 3)
for _ in range(3):
# Simulate server restart three times with same server ID
client.shutdown_server()
time.sleep(watch_interval * 3)
open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
time.sleep(watch_interval * 3)
# SImulate server update
client.shutdown_server()
time.sleep(watch_interval * 3)
open_server_process(port=port, socket=None)
time.sleep(watch_interval * 5)
shutdown_event.set()
watch_thread.join()
assert "on_disconnect" in events
assert "on_reconnected" in events
assert events[-1] == "on_updated"
def test_grpc_watch_thread_server_error():
port = find_free_port()
fixed_server_id = "fixed_id"
called = {}
def on_disconnect(location_name):
assert location_name == "test_location"
called["on_disconnect"] = True
def on_error(location_name):
assert location_name == "test_location"
called["on_error"] = True
def on_updated(location_name, new_server_id):
assert location_name == "test_location"
called["on_updated"] = new_server_id
def should_not_be_called(*args, **kwargs):
raise Exception("This method should not be called")
# Create initial server
server_process = open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1
max_reconnect_attempts = 3
shutdown_event, watch_thread = create_grpc_watch_thread(
"test_location",
client,
on_disconnect=on_disconnect,
on_reconnected=should_not_be_called,
on_updated=on_updated,
on_error=on_error,
watch_interval=watch_interval,
max_reconnect_attempts=max_reconnect_attempts,
)
watch_thread.start()
time.sleep(watch_interval * 3)
# Simulate restart failure
# Wait for reconnect attempts to exhaust and on_error callback to be called
interrupt_ipc_subprocess_pid(server_process.pid)
wait_for_condition(lambda: called.get("on_error"), watch_interval)
assert called["on_disconnect"]
assert called["on_error"]
assert not called.get("on_updated")
new_server_id = "new_server_id"
server_process = open_server_process(port=port,
socket=None,
fixed_server_id=new_server_id)
wait_for_condition(lambda: called.get("on_updated"), watch_interval)
shutdown_event.set()
watch_thread.join()
assert called["on_updated"] == new_server_id
