def test_load_via_env_var(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") subprocess_args = [ "dagster", "api", "grpc", "--python-file", python_file, ] with environ({ "DAGSTER_CLI_API_GRPC_HOST": "localhost", "DAGSTER_CLI_API_GRPC_PORT": str(port) }): process = subprocess.Popen( subprocess_args, stdout=subprocess.PIPE, ) try: wait_for_grpc_server( process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args) assert DagsterGrpcClient(port=port).ping("foobar") == "foobar" finally: process.terminate()
def test_load_with_error(capfd): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo_with_error.py") ipc_output_file = _get_ipc_output_file() process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--ipc-output-file", ipc_output_file, ], stdout=subprocess.PIPE, ) try: with pytest.raises(DagsterIPCProtocolError): wait_for_grpc_server(ipc_output_file) _, err = capfd.readouterr() assert "No module named" in err finally: process.terminate()
def test_load_with_error(capfd): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo_with_error.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, ] process = subprocess.Popen( subprocess_args, stdout=subprocess.PIPE, ) try: with pytest.raises(Exception): wait_for_grpc_server( process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args) process.wait() _, err = capfd.readouterr() assert "No module named" in err finally: if process.poll() is None: process.terminate()
def test_load_with_empty_working_directory(capfd): port = find_free_port() # File that will fail if working directory isn't set to default python_file = file_relative_path(__file__, "grpc_repo_with_local_import.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, ] with new_cwd(os.path.dirname(__file__)): process = subprocess.Popen( subprocess_args, stdout=subprocess.PIPE, ) try: wait_for_grpc_server( process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args ) assert DagsterGrpcClient(port=port).ping("foobar") == "foobar" finally: process.terminate() # indicating the working directory is empty fails port = find_free_port() subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--empty-working-directory", ] process = subprocess.Popen( subprocess_args, stdout=subprocess.PIPE, ) try: with pytest.raises(Exception): wait_for_grpc_server( process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args ) process.wait() _, err = capfd.readouterr() assert "No module named" in err finally: if process.poll() is None: process.terminate()
def test_lazy_load_via_env_var(): with environ({"DAGSTER_CLI_API_GRPC_LAZY_LOAD_USER_CODE": "1"}): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo_with_error.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, ] process = subprocess.Popen( subprocess_args, stdout=subprocess.PIPE, ) try: wait_for_grpc_server( process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args) list_repositories_response = deserialize_json_to_dagster_namedtuple( DagsterGrpcClient(port=port).list_repositories()) assert isinstance(list_repositories_response, SerializableErrorInfo) assert "No module named" in list_repositories_response.message finally: process.terminate()
def test_load_with_invalid_param(capfd): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") ipc_output_file = _get_ipc_output_file() process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--ipc-output-file", ipc_output_file, "--foo-param", "bar_value", ], stdout=subprocess.PIPE, ) try: with pytest.raises(DagsterIPCProtocolError): wait_for_grpc_server(process, ipc_output_file) finally: process.terminate() _, err = capfd.readouterr() assert "no such optio" in err
def test_lazy_load_with_error(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo_with_error.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--lazy-load-user-code", ] process = subprocess.Popen(subprocess_args, stdout=subprocess.PIPE) try: wait_for_grpc_server(process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args) list_repositories_response = deserialize_json_to_dagster_namedtuple( DagsterGrpcClient(port=port).list_repositories()) assert isinstance(list_repositories_response, SerializableErrorInfo) assert "No module named" in list_repositories_response.message finally: process.terminate()
def test_load_with_empty_working_directory(capfd): port = find_free_port() # File that will fail if working directory isn't set to default python_file = file_relative_path(__file__, "grpc_repo_with_local_import.py") with new_cwd(os.path.dirname(__file__)): ipc_output_file = _get_ipc_output_file() process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--ipc-output-file", ipc_output_file, ], stdout=subprocess.PIPE, ) try: wait_for_grpc_server(process, ipc_output_file) assert DagsterGrpcClient(port=port).ping("foobar") == "foobar" finally: process.terminate() # indicating the working directory is empty fails ipc_output_file = _get_ipc_output_file() process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--empty-working-directory", "--ipc-output-file", ipc_output_file, ], stdout=subprocess.PIPE, ) try: with pytest.raises(DagsterUserCodeProcessError): wait_for_grpc_server(process, ipc_output_file) process.wait() _, err = capfd.readouterr() assert "No module named" in err finally: if process.poll() is None: process.terminate()
def test_crash_during_load(): port = find_free_port() python_file = file_relative_path(__file__, "crashy_grpc_repo.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, ] process = subprocess.Popen( subprocess_args, stdout=subprocess.PIPE, ) try: with pytest.raises( Exception, match=re.escape( 'gRPC server exited with return code 123 while starting up with the command: "dagster api grpc --port' ), ): wait_for_grpc_server( process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args) finally: if process.poll() is None: process.terminate()
def test_sensor_timeout(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, ] process = subprocess.Popen( subprocess_args, stdout=subprocess.PIPE, ) try: wait_for_grpc_server( process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args ) client = DagsterGrpcClient(port=port) with instance_for_test() as instance: repo_origin = ExternalRepositoryOrigin( repository_location_origin=GrpcServerRepositoryLocationOrigin( port=port, host="localhost" ), repository_name="bar_repo", ) with pytest.raises(DagsterUserCodeUnreachableError) as exc_info: client.external_sensor_execution( sensor_execution_args=SensorExecutionArgs( repository_origin=repo_origin, instance_ref=instance.get_ref(), sensor_name="slow_sensor", last_completion_time=None, last_run_key=None, cursor=None, ), timeout=2, ) assert "Deadline Exceeded" in str(exc_info.getrepr()) # Call succeeds without the timeout client.external_sensor_execution( sensor_execution_args=SensorExecutionArgs( repository_origin=repo_origin, instance_ref=instance.get_ref(), sensor_name="slow_sensor", last_completion_time=None, last_run_key=None, cursor=None, ), ) finally: process.terminate()
def test_lazy_load_with_error(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo_with_error.py") ipc_output_file = _get_ipc_output_file() process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--lazy-load-user-code", "--ipc-output-file", ipc_output_file, ], stdout=subprocess.PIPE, ) try: wait_for_grpc_server(ipc_output_file) list_repositories_response = DagsterGrpcClient( port=port).list_repositories() assert isinstance(list_repositories_response, SerializableErrorInfo) assert "No module named" in list_repositories_response.message finally: process.terminate()
def test_load_with_invalid_param(capfd): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--foo-param", "bar_value", ] process = subprocess.Popen( subprocess_args, stdout=subprocess.PIPE, ) try: with pytest.raises( Exception, match='gRPC server exited with return code 2 while starting up with the command: "dagster api grpc --port', ): wait_for_grpc_server( process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args ) finally: process.terminate() _, err = capfd.readouterr() assert "no such option" in err
def test_load_grpc_server_python_env(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--use-python-environment-entry-point", ] process = subprocess.Popen(subprocess_args) try: client = DagsterGrpcClient(port=port, host="localhost") wait_for_grpc_server(process, client, subprocess_args) list_repositories_response = sync_list_repositories_grpc(client) assert list_repositories_response.entry_point == [ sys.executable, "-m", "dagster" ] assert list_repositories_response.executable_path == sys.executable finally: process.terminate()
def test_ping(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") ipc_output_file = _get_ipc_output_file() process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--ipc-output-file", ipc_output_file, ], stdout=subprocess.PIPE, ) try: wait_for_grpc_server(ipc_output_file) assert DagsterGrpcClient(port=port).ping("foobar") == "foobar" finally: process.terminate()
def test_streaming(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") ipc_output_file = _get_ipc_output_file() process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--ipc-output-file", ipc_output_file, ], stdout=subprocess.PIPE, ) try: wait_for_grpc_server(ipc_output_file) api_client = DagsterGrpcClient(port=port) results = [ result for result in api_client.streaming_ping(sequence_length=10, echo="foo") ] assert len(results) == 10 for sequence_number, result in enumerate(results): assert result["sequence_number"] == sequence_number assert result["echo"] == "foo" finally: process.terminate()
def test_lazy_load_via_env_var(): with environ({"DAGSTER_CLI_API_GRPC_LAZY_LOAD_USER_CODE": "1"}): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo_with_error.py") ipc_output_file = _get_ipc_output_file() process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--ipc-output-file", ipc_output_file, ], stdout=subprocess.PIPE, ) try: wait_for_grpc_server(process, ipc_output_file) list_repositories_response = DagsterGrpcClient(port=port).list_repositories() assert isinstance(list_repositories_response, SerializableErrorInfo) assert "No module named" in list_repositories_response.message finally: process.terminate()
def test_crash_during_load(): port = find_free_port() python_file = file_relative_path(__file__, "crashy_grpc_repo.py") ipc_output_file = _get_ipc_output_file() process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--ipc-output-file", ipc_output_file, ], stdout=subprocess.PIPE, ) try: with pytest.raises( Exception, match=re.escape( "Process exited with return code 123 while waiting for events" ), ): wait_for_grpc_server(process, ipc_output_file) finally: if process.poll() is None: process.terminate()
def test_sensor_timeout(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") ipc_output_file = _get_ipc_output_file() process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--ipc-output-file", ipc_output_file, ], stdout=subprocess.PIPE, ) try: wait_for_grpc_server(process, ipc_output_file) client = DagsterGrpcClient(port=port) with instance_for_test() as instance: repo_origin = ExternalRepositoryOrigin( repository_location_origin=GrpcServerRepositoryLocationOrigin( port=port, host="localhost" ), repository_name="bar_repo", ) with pytest.raises(Exception, match="Deadline Exceeded"): client.external_sensor_execution( sensor_execution_args=SensorExecutionArgs( repository_origin=repo_origin, instance_ref=instance.get_ref(), sensor_name="slow_sensor", last_completion_time=None, last_run_key=None, ), timeout=2, ) # Call succeeds without the timeout client.external_sensor_execution( sensor_execution_args=SensorExecutionArgs( repository_origin=repo_origin, instance_ref=instance.get_ref(), sensor_name="slow_sensor", last_completion_time=None, last_run_key=None, ), ) finally: process.terminate()
def test_ping(): port = find_free_port() python_file = file_relative_path(__file__, 'grpc_repo.py') process = subprocess.Popen( [ 'dagster', 'api', 'grpc', '--port', str(port), '--python-file', python_file ], stdout=subprocess.PIPE, ) try: wait_for_grpc_server(process) assert DagsterGrpcClient(port=port).ping('foobar') == 'foobar' finally: process.terminate()
def test_load_grpc_server(capfd): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, ] process = subprocess.Popen(subprocess_args) try: client = DagsterGrpcClient(port=port, host="localhost") wait_for_grpc_server(process, client, subprocess_args) assert client.ping("foobar") == "foobar" list_repositories_response = sync_list_repositories_grpc(client) assert list_repositories_response.entry_point == ["dagster"] assert list_repositories_response.executable_path == sys.executable subprocess.check_call( ["dagster", "api", "grpc-health-check", "--port", str(port)]) ssl_result = subprocess.run( # pylint:disable=subprocess-run-check [ "dagster", "api", "grpc-health-check", "--port", str(port), "--use-ssl" ]) assert ssl_result.returncode == 1 finally: process.terminate() out, _err = capfd.readouterr() assert f"Started Dagster code server for file {python_file} on port {port} in process" in out
def test_load_with_container_context(capfd): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") container_context = { "k8s": { "image_pull_policy": "Never", "image_pull_secrets": [{"name": "your_secret"}], } } subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--container-context", json.dumps(container_context), ] process = subprocess.Popen(subprocess_args) try: client = DagsterGrpcClient(port=port, host="localhost") wait_for_grpc_server(process, client, subprocess_args) assert client.ping("foobar") == "foobar" list_repositories_response = sync_list_repositories_grpc(client) assert list_repositories_response.entry_point == ["dagster"] assert list_repositories_response.executable_path == sys.executable assert list_repositories_response.container_context == container_context finally: process.terminate() out, _err = capfd.readouterr() assert f"Started Dagster code server for file {python_file} on port {port} in process" in out
def test_ping(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, ] process = subprocess.Popen(subprocess_args, stdout=subprocess.PIPE) try: wait_for_grpc_server( process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args ) assert DagsterGrpcClient(port=port).ping("foobar") == "foobar" finally: process.terminate()
def test_load_with_empty_working_directory(capfd): port = find_free_port() # File that will fail if working directory isn't set to default python_file = file_relative_path(__file__, "grpc_repo_with_local_import.py") with new_cwd(os.path.dirname(__file__)): process = subprocess.Popen( ["dagster", "api", "grpc", "--port", str(port), "--python-file", python_file], stdout=subprocess.PIPE, ) try: assert wait_for_grpc_server(process) assert DagsterGrpcClient(port=port).ping("foobar") == "foobar" finally: process.terminate() # indicating the working directory is empty fails process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--empty-working-directory", ], stdout=subprocess.PIPE, ) try: assert not wait_for_grpc_server(process) _, err = capfd.readouterr() assert "No module named" in err finally: process.terminate()
def test_load_timeout(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo_that_times_out.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, ] process = subprocess.Popen(subprocess_args, stdout=subprocess.PIPE) timeout_exception = None try: try: wait_for_grpc_server( process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args, timeout=0.01, ) assert False, "server should have timed out" except Exception as e: timeout_exception = e finally: process.terminate() assert "Timed out waiting for gRPC server to start" in str( timeout_exception) assert "Most recent connection error: " in str(timeout_exception) assert "StatusCode.UNAVAILABLE" in str(timeout_exception)
def test_streaming(): port = find_free_port() python_file = file_relative_path(__file__, 'grpc_repo.py') process = subprocess.Popen( [ 'dagster', 'api', 'grpc', '--port', str(port), '--python-file', python_file ], stdout=subprocess.PIPE, ) try: wait_for_grpc_server(process) api_client = DagsterGrpcClient(port=port) results = [ result for result in api_client.streaming_ping(sequence_length=10, echo='foo') ] assert len(results) == 10 for sequence_number, result in enumerate(results): assert result['sequence_number'] == sequence_number assert result['echo'] == 'foo' finally: process.terminate()
def test_load_with_error(capfd): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo_with_error.py") process = subprocess.Popen( ["dagster", "api", "grpc", "--port", str(port), "--python-file", python_file], stdout=subprocess.PIPE, ) try: assert not wait_for_grpc_server(process) _, err = capfd.readouterr() assert "No module named" in err finally: process.terminate()