Example #1
0
def test_grpc_watch_thread_server_complex_cycle_2():
    # Server goes down, comes back up as the same server three times, then goes away and comes
    # back as a new server

    port = find_free_port()
    fixed_server_id = "fixed_id"

    events = []
    called = {}

    def on_disconnect():
        events.append("on_disconnect")

    def on_reconnected():
        events.append("on_reconnected")

    def on_updated(_):
        events.append("on_updated")

    def on_error():
        called["on_error"] = True
        events.append("on_error")

    # Create initial server
    open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id)

    # Start watch thread
    client = DagsterGrpcClient(port=port)
    watch_interval = 1  # This is a faster watch interval than we would use in practice

    shutdown_event, watch_thread = create_grpc_watch_thread(
        client,
        on_disconnect=on_disconnect,
        on_reconnected=on_reconnected,
        on_updated=on_updated,
        on_error=on_error,
        watch_interval=watch_interval,
        max_reconnect_attempts=3,
    )
    watch_thread.start()
    time.sleep(watch_interval * 3)

    cycles = 3
    for x in range(1, cycles + 1):
        # Simulate server restart three times with same server ID
        client.shutdown_server()
        wait_for_condition(lambda: events.count("on_disconnect") == x, watch_interval)
        open_server_process(port=port, socket=None, fixed_server_id=fixed_server_id)
        wait_for_condition(lambda: events.count("on_reconnected") == x, watch_interval)

    # Simulate server failure
    client.shutdown_server()

    # Wait for reconnect attempts to exhaust and on_error callback to be called
    wait_for_condition(lambda: called.get("on_error"), watch_interval)

    shutdown_event.set()
    watch_thread.join()

    assert events[-1] == "on_error"
Example #2
0
def test_grpc_watch_thread_server_update():
    port = find_free_port()

    called = {}

    def on_updated():
        called["yup"] = True

    # Create initial server
    server_process = open_server_process(port=port, socket=None)

    try:
        # Start watch thread
        client = DagsterGrpcClient(port=port)
        watch_interval = 4
        shutdown_event, watch_thread = create_grpc_watch_thread(
            client, on_updated=on_updated, watch_interval=watch_interval)
        watch_thread.start()
        time.sleep(watch_interval * 2)
    finally:
        interrupt_ipc_subprocess_pid(server_process.pid)

    assert not called

    # Create updated server
    server_process = open_server_process(port=port, socket=None)

    try:
        time.sleep(watch_interval * 2)
    finally:
        interrupt_ipc_subprocess_pid(server_process.pid)

    shutdown_event.set()
    watch_thread.join()
    assert called
Example #3
0
def test_grpc_watch_thread_server_reconnect():
    port = find_free_port()
    fixed_server_id = "fixed_id"

    called = {}

    def on_disconnect(location_name):
        assert location_name == "test_location"
        called["on_disconnect"] = True

    def on_reconnected(location_name):
        assert location_name == "test_location"
        called["on_reconnected"] = True

    def should_not_be_called(*args, **kwargs):
        raise Exception("This method should not be called")

    # Create initial server
    server_process = open_server_process(port=port,
                                         socket=None,
                                         fixed_server_id=fixed_server_id)

    # Start watch thread
    client = DagsterGrpcClient(port=port)
    watch_interval = 1
    shutdown_event, watch_thread = create_grpc_watch_thread(
        "test_location",
        client,
        on_disconnect=on_disconnect,
        on_reconnected=on_reconnected,
        on_updated=should_not_be_called,
        on_error=should_not_be_called,
        watch_interval=watch_interval,
    )
    watch_thread.start()
    time.sleep(watch_interval * 3)

    # Wait three seconds, simulate restart server, wait three seconds
    interrupt_ipc_subprocess_pid(server_process.pid)
    wait_for_condition(lambda: called.get("on_disconnect"), watch_interval)

    server_process = open_server_process(port=port,
                                         socket=None,
                                         fixed_server_id=fixed_server_id)
    wait_for_condition(lambda: called.get("on_reconnected"), watch_interval)

    shutdown_event.set()
    watch_thread.join()
Example #4
0
def test_server_socket():
    with safe_tempfile_path() as skt:
        server_process = open_server_process(port=None, socket=skt)
        try:
            assert DagsterGrpcClient(socket=skt).ping("foobar") == "foobar"
        finally:
            interrupt_ipc_subprocess_pid(server_process.pid)
Example #5
0
def test_server_port():
    port = find_free_port()
    server_process = open_server_process(port=port, socket=None)
    assert server_process is not None

    try:
        assert DagsterGrpcClient(port=port).ping("foobar") == "foobar"
    finally:
        if server_process is not None:
            interrupt_ipc_subprocess_pid(server_process.pid)
Example #6
0
def test_fixed_server_id():
    port = find_free_port()
    server_process = open_server_process(port=port,
                                         socket=None,
                                         fixed_server_id="fixed_id")
    assert server_process is not None

    try:
        api_client = DagsterGrpcClient(port=port)
        assert api_client.get_server_id() == "fixed_id"
    finally:
        interrupt_ipc_subprocess_pid(server_process.pid)
Example #7
0
def test_grpc_watch_thread_server_update():
    port = find_free_port()

    called = {}

    def on_updated(location_name, _):
        assert location_name == "test_location"
        called["yup"] = True

    # Create initial server
    server_process = open_server_process(port=port, socket=None)

    try:
        # Start watch thread
        client = DagsterGrpcClient(port=port)
        watch_interval = 1
        shutdown_event, watch_thread = create_grpc_watch_thread(
            "test_location",
            client,
            on_updated=on_updated,
            watch_interval=watch_interval,
        )
        watch_thread.start()
        time.sleep(watch_interval * 3)
    finally:
        interrupt_ipc_subprocess_pid(server_process.pid)

    assert not called

    # Create updated server
    server_process = open_server_process(port=port, socket=None)

    try:
        wait_for_condition(lambda: called, interval=watch_interval)
    finally:
        interrupt_ipc_subprocess_pid(server_process.pid)

    shutdown_event.set()
    watch_thread.join()
    assert called
Example #8
0
def test_grpc_watch_thread_server_error():
    port = find_free_port()
    fixed_server_id = "fixed_id"

    called = {}

    def on_disconnect():
        called["on_disconnect"] = True

    def on_error():
        called["on_error"] = True

    def should_not_be_called():
        raise Exception("This method should not be called")

    # Create initial server
    server_process = open_server_process(port=port,
                                         socket=None,
                                         fixed_server_id=fixed_server_id)

    # Start watch thread
    client = DagsterGrpcClient(port=port)
    watch_interval = 1
    max_reconnect_attempts = 3
    shutdown_event, watch_thread = create_grpc_watch_thread(
        client,
        on_disconnect=on_disconnect,
        on_reconnected=should_not_be_called,
        on_updated=should_not_be_called,
        on_error=on_error,
        watch_interval=watch_interval,
        max_reconnect_attempts=max_reconnect_attempts,
    )
    watch_thread.start()

    # Wait three seconds, simulate restart failure
    time.sleep(watch_interval * 3)
    interrupt_ipc_subprocess_pid(server_process.pid)

    # Wait for reconnect attempts to exhaust and on_error callback to be called
    start_time = time.time()
    while not called.get("on_error"):
        if time.time() - start_time > 30:
            break

        time.sleep(1)

    shutdown_event.set()
    watch_thread.join()

    assert called["on_disconnect"]
    assert called["on_error"]
Example #9
0
def test_empty_executable_args():
    port = find_free_port()
    python_file = file_relative_path(__file__, "grpc_repo.py")
    loadable_target_origin = LoadableTargetOrigin(executable_path="",
                                                  python_file=python_file)
    # with an empty executable_path, the args change
    process = None
    try:
        process = open_server_process(
            port, socket=None, loadable_target_origin=loadable_target_origin)
        assert process.args[:3] == ["dagster", "api", "grpc"]
    finally:
        if process:
            process.terminate()
Example #10
0
def test_python_environment_args():
    port = find_free_port()
    python_file = file_relative_path(__file__, "grpc_repo.py")
    loadable_target_origin = LoadableTargetOrigin(
        executable_path=sys.executable, python_file=python_file)

    process = None
    try:
        process = open_server_process(
            port, socket=None, loadable_target_origin=loadable_target_origin)
        assert process.args[:5] == [
            sys.executable, "-m", "dagster", "api", "grpc"
        ]
    finally:
        if process:
            process.terminate()
Example #11
0
def create_server_process():
    port = find_free_port()
    server_process = open_server_process(port=port, socket=None)
    assert server_process is not None
    return port, server_process
Example #12
0
def test_grpc_watch_thread_server_complex_cycle():
    # Server goes down, comes back up as the same server three times, then goes away and comes
    # back as a new server

    port = find_free_port()
    fixed_server_id = "fixed_id"

    events = []

    def on_disconnect():
        events.append("on_disconnect")

    def on_reconnected():
        events.append("on_reconnected")

    def on_updated():
        events.append("on_updated")

    def on_error():
        events.append("on_error")

    # Create initial server
    open_server_process(port=port,
                        socket=None,
                        fixed_server_id=fixed_server_id)

    # Start watch thread
    client = DagsterGrpcClient(port=port)
    watch_interval = 1  # This is a faster watch interval than we would use in practice

    shutdown_event, watch_thread = create_grpc_watch_thread(
        client,
        on_disconnect=on_disconnect,
        on_reconnected=on_reconnected,
        on_updated=on_updated,
        on_error=on_error,
        watch_interval=watch_interval,
        max_reconnect_attempts=5,
    )
    watch_thread.start()
    time.sleep(watch_interval * 3)

    for _ in range(3):
        # Simulate server restart three times with same server ID
        client.shutdown_server()
        time.sleep(watch_interval * 3)
        open_server_process(port=port,
                            socket=None,
                            fixed_server_id=fixed_server_id)
        time.sleep(watch_interval * 3)

    # SImulate server update
    client.shutdown_server()
    time.sleep(watch_interval * 3)
    open_server_process(port=port, socket=None)
    time.sleep(watch_interval * 5)

    shutdown_event.set()
    watch_thread.join()

    assert "on_disconnect" in events
    assert "on_reconnected" in events
    assert events[-1] == "on_updated"
Example #13
0
def test_grpc_watch_thread_server_error():
    port = find_free_port()
    fixed_server_id = "fixed_id"

    called = {}

    def on_disconnect(location_name):
        assert location_name == "test_location"
        called["on_disconnect"] = True

    def on_error(location_name):
        assert location_name == "test_location"

        called["on_error"] = True

    def on_updated(location_name, new_server_id):
        assert location_name == "test_location"
        called["on_updated"] = new_server_id

    def should_not_be_called(*args, **kwargs):
        raise Exception("This method should not be called")

    # Create initial server
    server_process = open_server_process(port=port,
                                         socket=None,
                                         fixed_server_id=fixed_server_id)

    # Start watch thread
    client = DagsterGrpcClient(port=port)
    watch_interval = 1
    max_reconnect_attempts = 3
    shutdown_event, watch_thread = create_grpc_watch_thread(
        "test_location",
        client,
        on_disconnect=on_disconnect,
        on_reconnected=should_not_be_called,
        on_updated=on_updated,
        on_error=on_error,
        watch_interval=watch_interval,
        max_reconnect_attempts=max_reconnect_attempts,
    )
    watch_thread.start()
    time.sleep(watch_interval * 3)

    # Simulate restart failure
    # Wait for reconnect attempts to exhaust and on_error callback to be called
    interrupt_ipc_subprocess_pid(server_process.pid)
    wait_for_condition(lambda: called.get("on_error"), watch_interval)

    assert called["on_disconnect"]
    assert called["on_error"]
    assert not called.get("on_updated")

    new_server_id = "new_server_id"
    server_process = open_server_process(port=port,
                                         socket=None,
                                         fixed_server_id=new_server_id)

    wait_for_condition(lambda: called.get("on_updated"), watch_interval)

    shutdown_event.set()
    watch_thread.join()

    assert called["on_updated"] == new_server_id