def test_grpc_watch_thread_server_complex_cycle_2():
# Server goes down, comes back up as the same server three times, then goes away and comes
# back as a new server
port = find_free_port()
fixed_server_id = "fixed_id"
events = []
called = {}
def on_disconnect():
events.append("on_disconnect")
def on_reconnected():
events.append("on_reconnected")
def on_updated(_):
events.append("on_updated")
def on_error():
called["on_error"] = True
events.append("on_error")
# Create initial server
open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id)
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1 # This is a faster watch interval than we would use in practice
shutdown_event, watch_thread = create_grpc_watch_thread(
client,
on_disconnect=on_disconnect,
on_reconnected=on_reconnected,
on_updated=on_updated,
on_error=on_error,
watch_interval=watch_interval,
max_reconnect_attempts=3,
)
watch_thread.start()
time.sleep(watch_interval * 3)
cycles = 3
for x in range(1, cycles + 1):
# Simulate server restart three times with same server ID
client.shutdown_server()
wait_for_condition(lambda: events.count("on_disconnect") == x, watch_interval)
open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id)
wait_for_condition(lambda: events.count("on_reconnected") == x, watch_interval)
# Simulate server failure
client.shutdown_server()
# Wait for reconnect attempts to exhaust and on_error callback to be called
wait_for_condition(lambda: called.get("on_error"), watch_interval)
shutdown_event.set()
watch_thread.join()
assert events[-1] == "on_error"
def test_grpc_watch_thread_server_complex_cycle():
# Server goes down, comes back up as the same server three times, then goes away and comes
# back as a new server
port = find_free_port()
fixed_server_id = "fixed_id"
events = []
def on_disconnect():
events.append("on_disconnect")
def on_reconnected():
events.append("on_reconnected")
def on_updated():
events.append("on_updated")
def on_error():
events.append("on_error")
# Create initial server
open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1 # This is a faster watch interval than we would use in practice
shutdown_event, watch_thread = create_grpc_watch_thread(
client,
on_disconnect=on_disconnect,
on_reconnected=on_reconnected,
on_updated=on_updated,
on_error=on_error,
watch_interval=watch_interval,
max_reconnect_attempts=5,
)
watch_thread.start()
time.sleep(watch_interval * 3)
for _ in range(3):
# Simulate server restart three times with same server ID
client.shutdown_server()
time.sleep(watch_interval * 3)
open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
time.sleep(watch_interval * 3)
# SImulate server update
client.shutdown_server()
time.sleep(watch_interval * 3)
open_server_process(port=port, socket=None)
time.sleep(watch_interval * 5)
shutdown_event.set()
watch_thread.join()
assert "on_disconnect" in events
assert "on_reconnected" in events
assert events[-1] == "on_updated"
