def __init__(self, origin):
from dagster.grpc.client import client_heartbeat_thread
from dagster.grpc.server import GrpcServerProcess
self.grpc_server_process = None
self.client = None
self.heartbeat_shutdown_event = None
self.heartbeat_thread = None
self._origin = check.inst_param(
origin, "origin", ManagedGrpcPythonEnvRepositoryLocationOrigin)
loadable_target_origin = origin.loadable_target_origin
try:
self.grpc_server_process = GrpcServerProcess(
loadable_target_origin=loadable_target_origin,
heartbeat=True,
)
self.client = self.grpc_server_process.create_ephemeral_client()
self.heartbeat_shutdown_event = threading.Event()
self.heartbeat_thread = threading.Thread(
target=client_heartbeat_thread,
args=(
self.client,
self.heartbeat_shutdown_event,
),
name="grpc-client-heartbeat",
)
self.heartbeat_thread.daemon = True
self.heartbeat_thread.start()
list_repositories_response = sync_list_repositories_grpc(
self.client)
self.repository_code_pointer_dict = (
list_repositories_response.repository_code_pointer_dict)
self.container_image = self.client.get_current_image(
).current_image
external_repositories_list = sync_get_streaming_external_repositories_grpc(
self.client,
self,
)
self.external_repositories = {
repo.name: repo
for repo in external_repositories_list
}
except:
self.cleanup()
raise
def test_streaming_external_repositories_api_grpc():
with get_bar_repo_grpc_repository_location_handle(
) as repository_location_handle:
external_repos = sync_get_streaming_external_repositories_grpc(
repository_location_handle.client, repository_location_handle)
assert len(external_repos) == 1
external_repository = external_repos[0]
assert isinstance(external_repository, ExternalRepository)
assert external_repository.name == "bar_repo"
def test_giant_external_repository_streaming_grpc():
with get_giant_repo_grpc_repository_location_handle(
) as repository_location_handle:
# Using streaming allows the giant repo to load
external_repos = sync_get_streaming_external_repositories_grpc(
repository_location_handle.client, repository_location_handle)
assert len(external_repos) == 1
external_repository = external_repos[0]
assert isinstance(external_repository, ExternalRepository)
assert external_repository.name == "giant_repo"
def __init__(self, repository_location_handle):
check.param_invariant(
isinstance(repository_location_handle, GrpcServerRepositoryLocationHandle)
or isinstance(repository_location_handle, ManagedGrpcPythonEnvRepositoryLocationHandle),
"repository_location_handle",
)
self._handle = repository_location_handle
external_repositories_list = sync_get_streaming_external_repositories_grpc(
self._handle.client, self._handle,
)
self.external_repositories = {repo.name: repo for repo in external_repositories_list}
def __init__(
self,
origin,
host=None,
port=None,
socket=None,
server_id=None,
heartbeat=False,
watch_server=True,
):
from dagster.grpc.client import DagsterGrpcClient, client_heartbeat_thread
from dagster.grpc.server_watcher import create_grpc_watch_thread
self._origin = check.inst_param(origin, "origin",
RepositoryLocationOrigin)
if isinstance(self._origin, GrpcServerRepositoryLocationOrigin):
self._port = self.origin.port
self._socket = self.origin.socket
self._host = self.origin.host
self._use_ssl = bool(self.origin.use_ssl)
else:
self._port = check.opt_int_param(port, "port")
self._socket = check.opt_str_param(socket, "socket")
self._host = check.str_param(host, "host")
self._use_ssl = False
self._watch_thread_shutdown_event = None
self._watch_thread = None
self._heartbeat_shutdown_event = None
self._heartbeat_thread = None
self._heartbeat = check.bool_param(heartbeat, "heartbeat")
self._watch_server = check.bool_param(watch_server, "watch_server")
self.server_id = None
try:
self.client = DagsterGrpcClient(
port=self._port,
socket=self._socket,
host=self._host,
use_ssl=self._use_ssl,
)
list_repositories_response = sync_list_repositories_grpc(
self.client)
self.server_id = server_id if server_id else sync_get_server_id(
self.client)
self.repository_names = set(
symbol.repository_name
for symbol in list_repositories_response.repository_symbols)
if self._heartbeat:
self._heartbeat_shutdown_event = threading.Event()
self._heartbeat_thread = threading.Thread(
target=client_heartbeat_thread,
args=(
self.client,
self._heartbeat_shutdown_event,
),
name="grpc-client-heartbeat",
)
self._heartbeat_thread.daemon = True
self._heartbeat_thread.start()
if self._watch_server:
self._state_subscribers = []
self._watch_thread_shutdown_event, self._watch_thread = create_grpc_watch_thread(
self.client,
on_updated=lambda new_server_id: self.
_send_state_event_to_subscribers(
LocationStateChangeEvent(
LocationStateChangeEventType.LOCATION_UPDATED,
location_name=self.location_name,
message="Server has been updated.",
server_id=new_server_id,
)),
on_error=lambda: self._send_state_event_to_subscribers(
LocationStateChangeEvent(
LocationStateChangeEventType.LOCATION_ERROR,
location_name=self.location_name,
message=
"Unable to reconnect to server. You can reload the server once it is "
"reachable again",
)),
)
self._watch_thread.start()
self.executable_path = list_repositories_response.executable_path
self.repository_code_pointer_dict = (
list_repositories_response.repository_code_pointer_dict)
self.container_image = self._reload_current_image()
external_repositories_list = sync_get_streaming_external_repositories_grpc(
self.client,
self,
)
self.external_repositories = {
repo.name: repo
for repo in external_repositories_list
}
except:
self.cleanup()
raise
