def test_grpc_watch_thread_server_update(): port = find_free_port() called = {} def on_updated(): called["yup"] = True # Create initial server server_process = open_server_process(port=port, socket=None) try: # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 4 shutdown_event, watch_thread = create_grpc_watch_thread( client, on_updated=on_updated, watch_interval=watch_interval) watch_thread.start() time.sleep(watch_interval * 2) finally: interrupt_ipc_subprocess_pid(server_process.pid) assert not called # Create updated server server_process = open_server_process(port=port, socket=None) try: time.sleep(watch_interval * 2) finally: interrupt_ipc_subprocess_pid(server_process.pid) shutdown_event.set() watch_thread.join() assert called
def test_server_socket(): with safe_tempfile_path() as skt: server_process = open_server_process(port=None, socket=skt) try: assert DagsterGrpcClient(socket=skt).ping("foobar") == "foobar" finally: interrupt_ipc_subprocess_pid(server_process.pid)
def test_server_port(): port = find_free_port() server_process = open_server_process(port=port, socket=None) assert server_process is not None try: assert DagsterGrpcClient(port=port).ping("foobar") == "foobar" finally: if server_process is not None: interrupt_ipc_subprocess_pid(server_process.pid)
def test_fixed_server_id(): port = find_free_port() server_process = open_server_process(port=port, socket=None, fixed_server_id="fixed_id") assert server_process is not None try: api_client = DagsterGrpcClient(port=port) assert api_client.get_server_id() == "fixed_id" finally: interrupt_ipc_subprocess_pid(server_process.pid)
def test_grpc_watch_thread_server_error(): port = find_free_port() fixed_server_id = "fixed_id" called = {} def on_disconnect(): called["on_disconnect"] = True def on_error(): called["on_error"] = True def should_not_be_called(): raise Exception("This method should not be called") # Create initial server server_process = open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 1 max_reconnect_attempts = 3 shutdown_event, watch_thread = create_grpc_watch_thread( client, on_disconnect=on_disconnect, on_reconnected=should_not_be_called, on_updated=should_not_be_called, on_error=on_error, watch_interval=watch_interval, max_reconnect_attempts=max_reconnect_attempts, ) watch_thread.start() # Wait three seconds, simulate restart failure time.sleep(watch_interval * 3) interrupt_ipc_subprocess_pid(server_process.pid) # Wait for reconnect attempts to exhaust and on_error callback to be called start_time = time.time() while not called.get("on_error"): if time.time() - start_time > 30: break time.sleep(1) shutdown_event.set() watch_thread.join() assert called["on_disconnect"] assert called["on_error"]
def test_interrupt_ipc_subprocess_by_pid(): with safe_tempfile_path() as started_sentinel: with safe_tempfile_path() as interrupt_sentinel: sleepy_process = open_ipc_subprocess([ sys.executable, file_relative_path(__file__, "subprocess_with_interrupt_support.py"), started_sentinel, interrupt_sentinel, ]) wait_for_file(started_sentinel) interrupt_ipc_subprocess_pid(sleepy_process.pid) wait_for_file(interrupt_sentinel) with open(interrupt_sentinel, "r") as fd: assert fd.read().startswith("received_keyboard_interrupt")
def test_grpc_watch_thread_server_reconnect(): port = find_free_port() fixed_server_id = "fixed_id" called = {} def on_disconnect(location_name): assert location_name == "test_location" called["on_disconnect"] = True def on_reconnected(location_name): assert location_name == "test_location" called["on_reconnected"] = True def should_not_be_called(*args, **kwargs): raise Exception("This method should not be called") # Create initial server server_process = open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 1 shutdown_event, watch_thread = create_grpc_watch_thread( "test_location", client, on_disconnect=on_disconnect, on_reconnected=on_reconnected, on_updated=should_not_be_called, on_error=should_not_be_called, watch_interval=watch_interval, ) watch_thread.start() time.sleep(watch_interval * 3) # Wait three seconds, simulate restart server, wait three seconds interrupt_ipc_subprocess_pid(server_process.pid) wait_for_condition(lambda: called.get("on_disconnect"), watch_interval) server_process = open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) wait_for_condition(lambda: called.get("on_reconnected"), watch_interval) shutdown_event.set() watch_thread.join()
def terminate(self, run_id): check.str_param(run_id, 'run_id') process = self._get_process(run_id) if not process: return False if not process.is_alive(): return False # Pipeline execution machinery is set up to gracefully # terminate and report to instance on KeyboardInterrupt interrupt_ipc_subprocess_pid(process.pid) process.join() return True
def open_server_process(port, socket, python_executable_path=None): check.invariant((port or socket) and not (port and socket), 'Set only port or socket') python_executable_path = check.opt_str_param(python_executable_path, 'python_executable_path', default=sys.executable) server_process = open_ipc_subprocess( [python_executable_path, '-m', 'dagster.grpc'] + (['-p', str(port)] if port else []) + (['-f', socket] if socket else []), stdout=subprocess.PIPE, ) ready = _wait_for_grpc_server(server_process) if ready: return server_process else: if server_process.poll() is None: interrupt_ipc_subprocess_pid(server_process.pid) return None
def test_grpc_watch_thread_server_update(): port = find_free_port() called = {} def on_updated(location_name, _): assert location_name == "test_location" called["yup"] = True # Create initial server server_process = open_server_process(port=port, socket=None) try: # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 1 shutdown_event, watch_thread = create_grpc_watch_thread( "test_location", client, on_updated=on_updated, watch_interval=watch_interval, ) watch_thread.start() time.sleep(watch_interval * 3) finally: interrupt_ipc_subprocess_pid(server_process.pid) assert not called # Create updated server server_process = open_server_process(port=port, socket=None) try: wait_for_condition(lambda: called, interval=watch_interval) finally: interrupt_ipc_subprocess_pid(server_process.pid) shutdown_event.set() watch_thread.join() assert called
def test_detect_server_restart(): # Create first server and query ID port, server_process = create_server_process() try: api_client = DagsterGrpcClient(port=port) server_id_one = api_client.get_server_id() assert server_id_one finally: interrupt_ipc_subprocess_pid(server_process.pid) seven.wait_for_process(server_process, timeout=5) with pytest.raises(DagsterUserCodeUnreachableError): api_client.get_server_id() # Create second server and query ID port, server_process = create_server_process() try: api_client = DagsterGrpcClient(port=port) server_id_two = api_client.get_server_id() assert server_id_two finally: interrupt_ipc_subprocess_pid(server_process.pid) assert server_id_one != server_id_two
def test_detect_server_restart(): # Create first server and query ID port, server_process = create_server_process() try: api_client = DagsterGrpcClient(port=port) server_id_one = api_client.get_server_id() assert server_id_one finally: interrupt_ipc_subprocess_pid(server_process.pid) seven.wait_for_process(server_process, timeout=5) with pytest.raises(grpc._channel._InactiveRpcError): # pylint: disable=protected-access api_client.get_server_id() # Create second server and query ID port, server_process = create_server_process() try: api_client = DagsterGrpcClient(port=port) server_id_two = api_client.get_server_id() assert server_id_two finally: interrupt_ipc_subprocess_pid(server_process.pid) assert server_id_one != server_id_two
def test_grpc_watch_thread_server_error(): port = find_free_port() fixed_server_id = "fixed_id" called = {} def on_disconnect(location_name): assert location_name == "test_location" called["on_disconnect"] = True def on_error(location_name): assert location_name == "test_location" called["on_error"] = True def on_updated(location_name, new_server_id): assert location_name == "test_location" called["on_updated"] = new_server_id def should_not_be_called(*args, **kwargs): raise Exception("This method should not be called") # Create initial server server_process = open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id) # Start watch thread client = DagsterGrpcClient(port=port) watch_interval = 1 max_reconnect_attempts = 3 shutdown_event, watch_thread = create_grpc_watch_thread( "test_location", client, on_disconnect=on_disconnect, on_reconnected=should_not_be_called, on_updated=on_updated, on_error=on_error, watch_interval=watch_interval, max_reconnect_attempts=max_reconnect_attempts, ) watch_thread.start() time.sleep(watch_interval * 3) # Simulate restart failure # Wait for reconnect attempts to exhaust and on_error callback to be called interrupt_ipc_subprocess_pid(server_process.pid) wait_for_condition(lambda: called.get("on_error"), watch_interval) assert called["on_disconnect"] assert called["on_error"] assert not called.get("on_updated") new_server_id = "new_server_id" server_process = open_server_process(port=port, socket=None, fixed_server_id=new_server_id) wait_for_condition(lambda: called.get("on_updated"), watch_interval) shutdown_event.set() watch_thread.join() assert called["on_updated"] == new_server_id
def terminate_server_process(self): if self._server_process is not None: interrupt_ipc_subprocess_pid(self._server_process.pid) self._server_process = None