def test_grpc_watch_thread_server_complex_cycle_2(): # Server goes down, comes back up as the same server three times, then goes away and comes # back as a new server port = find_free_port() fixed_server_id = "fixed_id" events = [] called = {} def on_disconnect(): events.append("on_disconnect") def on_reconnected(): events.append("on_reconnected") def on_updated(_): events.append("on_updated") def on_error(): called["on_error"] = True events.append("on_error") # Create initial server open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 1 # This is a faster watch interval than we would use in practice shutdown_event, watch_thread = create_grpc_watch_thread( client, on_disconnect=on_disconnect, on_reconnected=on_reconnected, on_updated=on_updated, on_error=on_error, watch_interval=watch_interval, max_reconnect_attempts=3, ) watch_thread.start() time.sleep(watch_interval * 3) cycles = 3 for x in range(1, cycles + 1): # Simulate server restart three times with same server ID client.shutdown_server() wait_for_condition(lambda: events.count("on_disconnect") == x, watch_interval) open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) wait_for_condition(lambda: events.count("on_reconnected") == x, watch_interval) # Simulate server failure client.shutdown_server() # Wait for reconnect attempts to exhaust and on_error callback to be called wait_for_condition(lambda: called.get("on_error"), watch_interval) shutdown_event.set() watch_thread.join() assert events[-1] == "on_error"
def test_grpc_watch_thread_server_update(): port = find_free_port() called = {} def on_updated(): called["yup"] = True # Create initial server server_process = open_server_process(port=port, socket=None) try: # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 4 shutdown_event, watch_thread = create_grpc_watch_thread( client, on_updated=on_updated, watch_interval=watch_interval) watch_thread.start() time.sleep(watch_interval * 2) finally: interrupt_ipc_subprocess_pid(server_process.pid) assert not called # Create updated server server_process = open_server_process(port=port, socket=None) try: time.sleep(watch_interval * 2) finally: interrupt_ipc_subprocess_pid(server_process.pid) shutdown_event.set() watch_thread.join() assert called
def test_grpc_watch_thread_server_reconnect(): port = find_free_port() fixed_server_id = "fixed_id" called = {} def on_disconnect(location_name): assert location_name == "test_location" called["on_disconnect"] = True def on_reconnected(location_name): assert location_name == "test_location" called["on_reconnected"] = True def should_not_be_called(*args, **kwargs): raise Exception("This method should not be called") # Create initial server server_process = open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 1 shutdown_event, watch_thread = create_grpc_watch_thread( "test_location", client, on_disconnect=on_disconnect, on_reconnected=on_reconnected, on_updated=should_not_be_called, on_error=should_not_be_called, watch_interval=watch_interval, ) watch_thread.start() time.sleep(watch_interval * 3) # Wait three seconds, simulate restart server, wait three seconds interrupt_ipc_subprocess_pid(server_process.pid) wait_for_condition(lambda: called.get("on_disconnect"), watch_interval) server_process = open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) wait_for_condition(lambda: called.get("on_reconnected"), watch_interval) shutdown_event.set() watch_thread.join()
def test_server_socket(): with safe_tempfile_path() as skt: server_process = open_server_process(port=None, socket=skt) try: assert DagsterGrpcClient(socket=skt).ping("foobar") == "foobar" finally: interrupt_ipc_subprocess_pid(server_process.pid)
def test_server_port(): port = find_free_port() server_process = open_server_process(port=port, socket=None) assert server_process is not None try: assert DagsterGrpcClient(port=port).ping("foobar") == "foobar" finally: if server_process is not None: interrupt_ipc_subprocess_pid(server_process.pid)
def test_fixed_server_id(): port = find_free_port() server_process = open_server_process(port=port, socket=None, fixed_server_id="fixed_id") assert server_process is not None try: api_client = DagsterGrpcClient(port=port) assert api_client.get_server_id() == "fixed_id" finally: interrupt_ipc_subprocess_pid(server_process.pid)
def test_grpc_watch_thread_server_update(): port = find_free_port() called = {} def on_updated(location_name, _): assert location_name == "test_location" called["yup"] = True # Create initial server server_process = open_server_process(port=port, socket=None) try: # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 1 shutdown_event, watch_thread = create_grpc_watch_thread( "test_location", client, on_updated=on_updated, watch_interval=watch_interval, ) watch_thread.start() time.sleep(watch_interval * 3) finally: interrupt_ipc_subprocess_pid(server_process.pid) assert not called # Create updated server server_process = open_server_process(port=port, socket=None) try: wait_for_condition(lambda: called, interval=watch_interval) finally: interrupt_ipc_subprocess_pid(server_process.pid) shutdown_event.set() watch_thread.join() assert called
def test_grpc_watch_thread_server_error(): port = find_free_port() fixed_server_id = "fixed_id" called = {} def on_disconnect(): called["on_disconnect"] = True def on_error(): called["on_error"] = True def should_not_be_called(): raise Exception("This method should not be called") # Create initial server server_process = open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 1 max_reconnect_attempts = 3 shutdown_event, watch_thread = create_grpc_watch_thread( client, on_disconnect=on_disconnect, on_reconnected=should_not_be_called, on_updated=should_not_be_called, on_error=on_error, watch_interval=watch_interval, max_reconnect_attempts=max_reconnect_attempts, ) watch_thread.start() # Wait three seconds, simulate restart failure time.sleep(watch_interval * 3) interrupt_ipc_subprocess_pid(server_process.pid) # Wait for reconnect attempts to exhaust and on_error callback to be called start_time = time.time() while not called.get("on_error"): if time.time() - start_time > 30: break time.sleep(1) shutdown_event.set() watch_thread.join() assert called["on_disconnect"] assert called["on_error"]
def test_empty_executable_args(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") loadable_target_origin = LoadableTargetOrigin(executable_path="", python_file=python_file) # with an empty executable_path, the args change process = None try: process = open_server_process( port, socket=None, loadable_target_origin=loadable_target_origin) assert process.args[:3] == ["dagster", "api", "grpc"] finally: if process: process.terminate()
def test_python_environment_args(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") loadable_target_origin = LoadableTargetOrigin( executable_path=sys.executable, python_file=python_file) process = None try: process = open_server_process( port, socket=None, loadable_target_origin=loadable_target_origin) assert process.args[:5] == [ sys.executable, "-m", "dagster", "api", "grpc" ] finally: if process: process.terminate()
def create_server_process(): port = find_free_port() server_process = open_server_process(port=port, socket=None) assert server_process is not None return port, server_process
def test_grpc_watch_thread_server_complex_cycle(): # Server goes down, comes back up as the same server three times, then goes away and comes # back as a new server port = find_free_port() fixed_server_id = "fixed_id" events = [] def on_disconnect(): events.append("on_disconnect") def on_reconnected(): events.append("on_reconnected") def on_updated(): events.append("on_updated") def on_error(): events.append("on_error") # Create initial server open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 1 # This is a faster watch interval than we would use in practice shutdown_event, watch_thread = create_grpc_watch_thread( client, on_disconnect=on_disconnect, on_reconnected=on_reconnected, on_updated=on_updated, on_error=on_error, watch_interval=watch_interval, max_reconnect_attempts=5, ) watch_thread.start() time.sleep(watch_interval * 3) for _ in range(3): # Simulate server restart three times with same server ID client.shutdown_server() time.sleep(watch_interval * 3) open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) time.sleep(watch_interval * 3) # SImulate server update client.shutdown_server() time.sleep(watch_interval * 3) open_server_process(port=port, socket=None) time.sleep(watch_interval * 5) shutdown_event.set() watch_thread.join() assert "on_disconnect" in events assert "on_reconnected" in events assert events[-1] == "on_updated"
def test_grpc_watch_thread_server_error(): port = find_free_port() fixed_server_id = "fixed_id" called = {} def on_disconnect(location_name): assert location_name == "test_location" called["on_disconnect"] = True def on_error(location_name): assert location_name == "test_location" called["on_error"] = True def on_updated(location_name, new_server_id): assert location_name == "test_location" called["on_updated"] = new_server_id def should_not_be_called(*args, **kwargs): raise Exception("This method should not be called") # Create initial server server_process = open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 1 max_reconnect_attempts = 3 shutdown_event, watch_thread = create_grpc_watch_thread( "test_location", client, on_disconnect=on_disconnect, on_reconnected=should_not_be_called, on_updated=on_updated, on_error=on_error, watch_interval=watch_interval, max_reconnect_attempts=max_reconnect_attempts, ) watch_thread.start() time.sleep(watch_interval * 3) # Simulate restart failure # Wait for reconnect attempts to exhaust and on_error callback to be called interrupt_ipc_subprocess_pid(server_process.pid) wait_for_condition(lambda: called.get("on_error"), watch_interval) assert called["on_disconnect"] assert called["on_error"] assert not called.get("on_updated") new_server_id = "new_server_id" server_process = open_server_process(port=port, socket=None, fixed_server_id=new_server_id) wait_for_condition(lambda: called.get("on_updated"), watch_interval) shutdown_event.set() watch_thread.join() assert called["on_updated"] == new_server_id