def test_grpc_watch_thread_server_update():
port = find_free_port()
called = {}
def on_updated():
called["yup"] = True
# Create initial server
server_process = open_server_process(port=port, socket=None)
try:
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 4
shutdown_event, watch_thread = create_grpc_watch_thread(
client, on_updated=on_updated, watch_interval=watch_interval)
watch_thread.start()
time.sleep(watch_interval * 2)
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
assert not called
# Create updated server
server_process = open_server_process(port=port, socket=None)
try:
time.sleep(watch_interval * 2)
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
shutdown_event.set()
watch_thread.join()
assert called
def test_server_socket():
with safe_tempfile_path() as skt:
server_process = open_server_process(port=None, socket=skt)
try:
assert DagsterGrpcClient(socket=skt).ping("foobar") == "foobar"
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
def test_server_port():
port = find_free_port()
server_process = open_server_process(port=port, socket=None)
assert server_process is not None
try:
assert DagsterGrpcClient(port=port).ping("foobar") == "foobar"
finally:
if server_process is not None:
interrupt_ipc_subprocess_pid(server_process.pid)
def test_fixed_server_id():
port = find_free_port()
server_process = open_server_process(port=port,
socket=None,
fixed_server_id="fixed_id")
assert server_process is not None
try:
api_client = DagsterGrpcClient(port=port)
assert api_client.get_server_id() == "fixed_id"
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
def test_grpc_watch_thread_server_error():
port = find_free_port()
fixed_server_id = "fixed_id"
called = {}
def on_disconnect():
called["on_disconnect"] = True
def on_error():
called["on_error"] = True
def should_not_be_called():
raise Exception("This method should not be called")
# Create initial server
server_process = open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1
max_reconnect_attempts = 3
shutdown_event, watch_thread = create_grpc_watch_thread(
client,
on_disconnect=on_disconnect,
on_reconnected=should_not_be_called,
on_updated=should_not_be_called,
on_error=on_error,
watch_interval=watch_interval,
max_reconnect_attempts=max_reconnect_attempts,
)
watch_thread.start()
# Wait three seconds, simulate restart failure
time.sleep(watch_interval * 3)
interrupt_ipc_subprocess_pid(server_process.pid)
# Wait for reconnect attempts to exhaust and on_error callback to be called
start_time = time.time()
while not called.get("on_error"):
if time.time() - start_time > 30:
break
time.sleep(1)
shutdown_event.set()
watch_thread.join()
assert called["on_disconnect"]
assert called["on_error"]
def test_interrupt_ipc_subprocess_by_pid():
with safe_tempfile_path() as started_sentinel:
with safe_tempfile_path() as interrupt_sentinel:
sleepy_process = open_ipc_subprocess([
sys.executable,
file_relative_path(__file__,
"subprocess_with_interrupt_support.py"),
started_sentinel,
interrupt_sentinel,
])
wait_for_file(started_sentinel)
interrupt_ipc_subprocess_pid(sleepy_process.pid)
wait_for_file(interrupt_sentinel)
with open(interrupt_sentinel, "r") as fd:
assert fd.read().startswith("received_keyboard_interrupt")
def test_grpc_watch_thread_server_reconnect():
port = find_free_port()
fixed_server_id = "fixed_id"
called = {}
def on_disconnect(location_name):
assert location_name == "test_location"
called["on_disconnect"] = True
def on_reconnected(location_name):
assert location_name == "test_location"
called["on_reconnected"] = True
def should_not_be_called(*args, **kwargs):
raise Exception("This method should not be called")
# Create initial server
server_process = open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1
shutdown_event, watch_thread = create_grpc_watch_thread(
"test_location",
client,
on_disconnect=on_disconnect,
on_reconnected=on_reconnected,
on_updated=should_not_be_called,
on_error=should_not_be_called,
watch_interval=watch_interval,
)
watch_thread.start()
time.sleep(watch_interval * 3)
# Wait three seconds, simulate restart server, wait three seconds
interrupt_ipc_subprocess_pid(server_process.pid)
wait_for_condition(lambda: called.get("on_disconnect"), watch_interval)
server_process = open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
wait_for_condition(lambda: called.get("on_reconnected"), watch_interval)
shutdown_event.set()
watch_thread.join()
def terminate(self, run_id):
check.str_param(run_id, 'run_id')
process = self._get_process(run_id)
if not process:
return False
if not process.is_alive():
return False
# Pipeline execution machinery is set up to gracefully
# terminate and report to instance on KeyboardInterrupt
interrupt_ipc_subprocess_pid(process.pid)
process.join()
return True
def open_server_process(port, socket, python_executable_path=None):
check.invariant((port or socket) and not (port and socket),
'Set only port or socket')
python_executable_path = check.opt_str_param(python_executable_path,
'python_executable_path',
default=sys.executable)
server_process = open_ipc_subprocess(
[python_executable_path, '-m', 'dagster.grpc'] +
(['-p', str(port)] if port else []) +
(['-f', socket] if socket else []),
stdout=subprocess.PIPE,
)
ready = _wait_for_grpc_server(server_process)
if ready:
return server_process
else:
if server_process.poll() is None:
interrupt_ipc_subprocess_pid(server_process.pid)
return None
def test_grpc_watch_thread_server_update():
port = find_free_port()
called = {}
def on_updated(location_name, _):
assert location_name == "test_location"
called["yup"] = True
# Create initial server
server_process = open_server_process(port=port, socket=None)
try:
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1
shutdown_event, watch_thread = create_grpc_watch_thread(
"test_location",
client,
on_updated=on_updated,
watch_interval=watch_interval,
)
watch_thread.start()
time.sleep(watch_interval * 3)
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
assert not called
# Create updated server
server_process = open_server_process(port=port, socket=None)
try:
wait_for_condition(lambda: called, interval=watch_interval)
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
shutdown_event.set()
watch_thread.join()
assert called
def test_detect_server_restart():
# Create first server and query ID
port, server_process = create_server_process()
try:
api_client = DagsterGrpcClient(port=port)
server_id_one = api_client.get_server_id()
assert server_id_one
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
seven.wait_for_process(server_process, timeout=5)
with pytest.raises(DagsterUserCodeUnreachableError):
api_client.get_server_id()
# Create second server and query ID
port, server_process = create_server_process()
try:
api_client = DagsterGrpcClient(port=port)
server_id_two = api_client.get_server_id()
assert server_id_two
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
assert server_id_one != server_id_two
def test_detect_server_restart():
# Create first server and query ID
port, server_process = create_server_process()
try:
api_client = DagsterGrpcClient(port=port)
server_id_one = api_client.get_server_id()
assert server_id_one
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
seven.wait_for_process(server_process, timeout=5)
with pytest.raises(grpc._channel._InactiveRpcError): # pylint: disable=protected-access
api_client.get_server_id()
# Create second server and query ID
port, server_process = create_server_process()
try:
api_client = DagsterGrpcClient(port=port)
server_id_two = api_client.get_server_id()
assert server_id_two
finally:
interrupt_ipc_subprocess_pid(server_process.pid)
assert server_id_one != server_id_two
def test_grpc_watch_thread_server_error():
port = find_free_port()
fixed_server_id = "fixed_id"
called = {}
def on_disconnect(location_name):
assert location_name == "test_location"
called["on_disconnect"] = True
def on_error(location_name):
assert location_name == "test_location"
called["on_error"] = True
def on_updated(location_name, new_server_id):
assert location_name == "test_location"
called["on_updated"] = new_server_id
def should_not_be_called(*args, **kwargs):
raise Exception("This method should not be called")
# Create initial server
server_process = open_server_process(port=port,
socket=None,
fixed_server_id=fixed_server_id)
# Start watch thread
client = DagsterGrpcClient(port=port)
watch_interval = 1
max_reconnect_attempts = 3
shutdown_event, watch_thread = create_grpc_watch_thread(
"test_location",
client,
on_disconnect=on_disconnect,
on_reconnected=should_not_be_called,
on_updated=on_updated,
on_error=on_error,
watch_interval=watch_interval,
max_reconnect_attempts=max_reconnect_attempts,
)
watch_thread.start()
time.sleep(watch_interval * 3)
# Simulate restart failure
# Wait for reconnect attempts to exhaust and on_error callback to be called
interrupt_ipc_subprocess_pid(server_process.pid)
wait_for_condition(lambda: called.get("on_error"), watch_interval)
assert called["on_disconnect"]
assert called["on_error"]
assert not called.get("on_updated")
new_server_id = "new_server_id"
server_process = open_server_process(port=port,
socket=None,
fixed_server_id=new_server_id)
wait_for_condition(lambda: called.get("on_updated"), watch_interval)
shutdown_event.set()
watch_thread.join()
assert called["on_updated"] == new_server_id
def terminate_server_process(self):
if self._server_process is not None:
interrupt_ipc_subprocess_pid(self._server_process.pid)
self._server_process = None
