def create_process_bound_grpc_server_location(loadable_target_origin, location_name):
from dagster.grpc.server import GrpcServerProcess
server = GrpcServerProcess(loadable_target_origin=loadable_target_origin)
client = server.create_client()
list_repositories_response = sync_list_repositories_grpc(client)
code_pointer_dict = list_repositories_response.repository_code_pointer_dict
return ManagedGrpcPythonEnvRepositoryLocationHandle(
executable_path=list_repositories_response.executable_path,
location_name=location_name
if location_name
else _assign_python_env_location_name(code_pointer_dict),
repository_code_pointer_dict=code_pointer_dict,
client=client,
grpc_server_process=server,
)
def test_load_grpc_server(capfd):
port = find_free_port()
python_file = file_relative_path(__file__, "grpc_repo.py")
subprocess_args = [
"dagster",
"api",
"grpc",
"--port",
str(port),
"--python-file",
python_file,
]
process = subprocess.Popen(subprocess_args)
try:
client = DagsterGrpcClient(port=port, host="localhost")
wait_for_grpc_server(process, client, subprocess_args)
assert client.ping("foobar") == "foobar"
list_repositories_response = sync_list_repositories_grpc(client)
assert list_repositories_response.entry_point == ["dagster"]
assert list_repositories_response.executable_path == sys.executable
subprocess.check_call(
["dagster", "api", "grpc-health-check", "--port",
str(port)])
ssl_result = subprocess.run( # pylint:disable=subprocess-run-check
[
"dagster", "api", "grpc-health-check", "--port",
str(port), "--use-ssl"
])
assert ssl_result.returncode == 1
finally:
process.terminate()
out, _err = capfd.readouterr()
assert f"Started Dagster code server for file {python_file} on port {port} in process" in out
def test_load_with_container_context(capfd):
port = find_free_port()
python_file = file_relative_path(__file__, "grpc_repo.py")
container_context = {
"k8s": {
"image_pull_policy": "Never",
"image_pull_secrets": [{"name": "your_secret"}],
}
}
subprocess_args = [
"dagster",
"api",
"grpc",
"--port",
str(port),
"--python-file",
python_file,
"--container-context",
json.dumps(container_context),
]
process = subprocess.Popen(subprocess_args)
try:
client = DagsterGrpcClient(port=port, host="localhost")
wait_for_grpc_server(process, client, subprocess_args)
assert client.ping("foobar") == "foobar"
list_repositories_response = sync_list_repositories_grpc(client)
assert list_repositories_response.entry_point == ["dagster"]
assert list_repositories_response.executable_path == sys.executable
assert list_repositories_response.container_context == container_context
finally:
process.terminate()
out, _err = capfd.readouterr()
assert f"Started Dagster code server for file {python_file} on port {port} in process" in out
def __init__(self, origin):
from dagster.grpc.client import client_heartbeat_thread
from dagster.grpc.server import GrpcServerProcess
self.client = None
self.origin = check.inst_param(
origin, "origin", ManagedGrpcPythonEnvRepositoryLocationOrigin
)
loadable_target_origin = origin.loadable_target_origin
self.grpc_server_process = GrpcServerProcess(
loadable_target_origin=loadable_target_origin,
max_workers=2,
heartbeat=True,
lazy_load_user_code=True,
)
try:
self.client = self.grpc_server_process.create_ephemeral_client()
self.heartbeat_shutdown_event = threading.Event()
self.heartbeat_thread = threading.Thread(
target=client_heartbeat_thread,
args=(
self.client,
self.heartbeat_shutdown_event,
),
name="grpc-client-heartbeat",
)
self.heartbeat_thread.daemon = True
self.heartbeat_thread.start()
list_repositories_response = sync_list_repositories_grpc(self.client)
except:
self.cleanup()
raise
self.repository_code_pointer_dict = list_repositories_response.repository_code_pointer_dict
self.container_image = self.client.get_current_image().current_image
def __init__(self, origin):
from dagster.grpc.client import DagsterGrpcClient
self.origin = check.inst_param(origin, "origin",
GrpcServerRepositoryLocationOrigin)
port = self.origin.port
socket = self.origin.socket
host = self.origin.host
self.client = DagsterGrpcClient(port=port, socket=socket, host=host)
list_repositories_response = sync_list_repositories_grpc(self.client)
self.repository_names = set(
symbol.repository_name
for symbol in list_repositories_response.repository_symbols)
self.executable_path = list_repositories_response.executable_path
self.repository_code_pointer_dict = list_repositories_response.repository_code_pointer_dict
def test_sync_list_container_grpc(docker_grpc_client):
response = sync_list_repositories_grpc(docker_grpc_client)
loadable_repo_symbols = response.repository_symbols
assert docker_grpc_client.get_current_image().current_image == get_test_project_docker_image()
assert isinstance(loadable_repo_symbols, list)
assert len(loadable_repo_symbols) == 1
assert isinstance(loadable_repo_symbols[0], LoadableRepositorySymbol)
symbol = loadable_repo_symbols[0]
assert symbol.repository_name == "bar_repo"
assert symbol.attribute == "bar_repo"
executable_path = response.executable_path
assert executable_path
repository_code_pointer_dict = response.repository_code_pointer_dict
assert "bar_repo" in repository_code_pointer_dict
assert isinstance(repository_code_pointer_dict["bar_repo"], FileCodePointer)
assert repository_code_pointer_dict["bar_repo"].python_file.endswith("repo.py")
assert repository_code_pointer_dict["bar_repo"].fn_name == "bar_repo"
def test_sync_list_container_grpc(docker_grpc_client):
response = sync_list_repositories_grpc(docker_grpc_client)
loadable_repo_symbols = response.repository_symbols
assert isinstance(loadable_repo_symbols, list)
assert len(loadable_repo_symbols) == 1
assert isinstance(loadable_repo_symbols[0], LoadableRepositorySymbol)
symbol = loadable_repo_symbols[0]
assert symbol.repository_name == 'bar_repo'
assert symbol.attribute == 'bar_repo'
executable_path = response.executable_path
assert executable_path
repository_code_pointer_dict = response.repository_code_pointer_dict
assert 'bar_repo' in repository_code_pointer_dict
assert isinstance(repository_code_pointer_dict['bar_repo'],
FileCodePointer)
assert repository_code_pointer_dict['bar_repo'].python_file.endswith(
"repo.py")
assert repository_code_pointer_dict['bar_repo'].fn_name == 'bar_repo'
def __init__(
self,
origin,
host=None,
port=None,
socket=None,
server_id=None,
heartbeat=False,
watch_server=True,
):
from dagster.grpc.client import DagsterGrpcClient, client_heartbeat_thread
from dagster.grpc.server_watcher import create_grpc_watch_thread
self._origin = check.inst_param(origin, "origin",
RepositoryLocationOrigin)
if isinstance(self._origin, GrpcServerRepositoryLocationOrigin):
self._port = self.origin.port
self._socket = self.origin.socket
self._host = self.origin.host
self._use_ssl = bool(self.origin.use_ssl)
else:
self._port = check.opt_int_param(port, "port")
self._socket = check.opt_str_param(socket, "socket")
self._host = check.str_param(host, "host")
self._use_ssl = False
self._watch_thread_shutdown_event = None
self._watch_thread = None
self._heartbeat_shutdown_event = None
self._heartbeat_thread = None
self._heartbeat = check.bool_param(heartbeat, "heartbeat")
self._watch_server = check.bool_param(watch_server, "watch_server")
self.server_id = None
self._external_repositories_data = None
try:
self.client = DagsterGrpcClient(
port=self._port,
socket=self._socket,
host=self._host,
use_ssl=self._use_ssl,
)
list_repositories_response = sync_list_repositories_grpc(
self.client)
self.server_id = server_id if server_id else sync_get_server_id(
self.client)
self.repository_names = set(
symbol.repository_name
for symbol in list_repositories_response.repository_symbols)
if self._heartbeat:
self._heartbeat_shutdown_event = threading.Event()
self._heartbeat_thread = threading.Thread(
target=client_heartbeat_thread,
args=(
self.client,
self._heartbeat_shutdown_event,
),
name="grpc-client-heartbeat",
)
self._heartbeat_thread.daemon = True
self._heartbeat_thread.start()
if self._watch_server:
self._state_subscribers = []
self._watch_thread_shutdown_event, self._watch_thread = create_grpc_watch_thread(
self.client,
on_updated=lambda new_server_id: self.
_send_state_event_to_subscribers(
LocationStateChangeEvent(
LocationStateChangeEventType.LOCATION_UPDATED,
location_name=self.location_name,
message="Server has been updated.",
server_id=new_server_id,
)),
on_error=lambda: self._send_state_event_to_subscribers(
LocationStateChangeEvent(
LocationStateChangeEventType.LOCATION_ERROR,
location_name=self.location_name,
message=
"Unable to reconnect to server. You can reload the server once it is "
"reachable again",
)),
)
self._watch_thread.start()
self.executable_path = list_repositories_response.executable_path
self.repository_code_pointer_dict = (
list_repositories_response.repository_code_pointer_dict)
self.container_image = self._reload_current_image()
self._external_repositories_data = sync_get_streaming_external_repositories_data_grpc(
self.client,
self,
)
except:
self.cleanup()
raise
def __init__(
self,
origin: RepositoryLocationOrigin,
host: Optional[str] = None,
port: Optional[int] = None,
socket: Optional[str] = None,
server_id: Optional[str] = None,
heartbeat: Optional[bool] = False,
watch_server: Optional[bool] = True,
grpc_server_registry: Optional[GrpcServerRegistry] = None,
):
from dagster.grpc.client import DagsterGrpcClient, client_heartbeat_thread
self._origin = check.inst_param(origin, "origin",
RepositoryLocationOrigin)
self.grpc_server_registry = check.opt_inst_param(
grpc_server_registry, "grpc_server_registry", GrpcServerRegistry)
if isinstance(self.origin, GrpcServerRepositoryLocationOrigin):
self._port = self.origin.port
self._socket = self.origin.socket
self._host = self.origin.host
self._use_ssl = bool(self.origin.use_ssl)
else:
self._port = check.opt_int_param(port, "port")
self._socket = check.opt_str_param(socket, "socket")
self._host = check.str_param(host, "host")
self._use_ssl = False
self._watch_thread_shutdown_event = None
self._watch_thread = None
self._heartbeat_shutdown_event = None
self._heartbeat_thread = None
self._heartbeat = check.bool_param(heartbeat, "heartbeat")
self._watch_server = check.bool_param(watch_server, "watch_server")
self.server_id = None
self._external_repositories_data = None
self._executable_path = None
self._container_image = None
self._repository_code_pointer_dict = None
try:
self.client = DagsterGrpcClient(
port=self._port,
socket=self._socket,
host=self._host,
use_ssl=self._use_ssl,
)
list_repositories_response = sync_list_repositories_grpc(
self.client)
self.server_id = server_id if server_id else sync_get_server_id(
self.client)
self.repository_names = set(
symbol.repository_name
for symbol in list_repositories_response.repository_symbols)
if self._heartbeat:
self._heartbeat_shutdown_event = threading.Event()
self._heartbeat_thread = threading.Thread(
target=client_heartbeat_thread,
args=(
self.client,
self._heartbeat_shutdown_event,
),
name="grpc-client-heartbeat",
)
self._heartbeat_thread.daemon = True
self._heartbeat_thread.start()
self._executable_path = list_repositories_response.executable_path
self._repository_code_pointer_dict = (
list_repositories_response.repository_code_pointer_dict)
self._container_image = self._reload_current_image()
self._external_repositories_data = sync_get_streaming_external_repositories_data_grpc(
self.client,
self,
)
self.external_repositories = {
repo_name: ExternalRepository(
repo_data,
RepositoryHandle(
repository_name=repo_name,
repository_location=self,
),
)
for repo_name, repo_data in
self._external_repositories_data.items()
}
except:
self.cleanup()
raise
