def __init__(self, origin: InProcessRepositoryLocationOrigin): from dagster.grpc.server import LoadedRepositories self._origin = check.inst_param(origin, "origin", InProcessRepositoryLocationOrigin) loadable_target_origin = self._origin.loadable_target_origin self._loaded_repositories = LoadedRepositories( loadable_target_origin, self._origin.entry_point) self._repository_code_pointer_dict = self._loaded_repositories.code_pointers_by_repo_name self._recon_repos = { repo_name: self._loaded_repositories.get_recon_repo(repo_name) for repo_name in self._repository_code_pointer_dict } self._repositories = {} for repo_name in self._repository_code_pointer_dict: recon_repo = self._loaded_repositories.get_recon_repo(repo_name) repo_def = recon_repo.get_definition() self._repositories[repo_name] = external_repo_from_def( repo_def, RepositoryHandle(repository_name=repo_name, repository_location=self), )
def _external_pipeline_from_def(pipeline_def, solid_subset=None): recon_pipeline = reconstructable(pipeline_def) recon_repo = recon_pipeline.repository repo_def = recon_repo.get_definition() location_handle = LocationHandle('test', recon_repo.pointer) repository_handle = RepositoryHandle(repo_def.name, location_handle) return external_pipeline_from_recon_pipeline( reconstructable(pipeline_def), solid_subset=solid_subset, repository_handle=repository_handle, )
def _external_pipeline_from_def(pipeline_def, solid_selection=None): recon_pipeline = reconstructable(pipeline_def) recon_repo = recon_pipeline.repository repo_def = recon_repo.get_definition() location_handle = RepositoryLocationHandle.create_in_process_location(recon_repo.pointer) repository_handle = RepositoryHandle( repository_name=repo_def.name, repository_location_handle=location_handle, ) return external_pipeline_from_recon_pipeline( reconstructable(pipeline_def), solid_selection=solid_selection, repository_handle=repository_handle, )
def __init__(self, origin: InProcessRepositoryLocationOrigin): self._origin = check.inst_param(origin, "origin", InProcessRepositoryLocationOrigin) self._recon_repo = self._origin.recon_repo repo_def = self._recon_repo.get_definition() pointer = self._recon_repo.pointer self._repository_code_pointer_dict = {repo_def.name: pointer} def_name = repo_def.name self._external_repo = external_repo_from_def( repo_def, RepositoryHandle(repository_name=def_name, repository_location=self), ) self._repositories = {self._external_repo.name: self._external_repo}
def __init__( self, origin: RepositoryLocationOrigin, host: Optional[str] = None, port: Optional[int] = None, socket: Optional[str] = None, server_id: Optional[str] = None, heartbeat: Optional[bool] = False, watch_server: Optional[bool] = True, grpc_server_registry: Optional[GrpcServerRegistry] = None, ): from dagster.grpc.client import DagsterGrpcClient, client_heartbeat_thread self._origin = check.inst_param(origin, "origin", RepositoryLocationOrigin) self.grpc_server_registry = check.opt_inst_param( grpc_server_registry, "grpc_server_registry", GrpcServerRegistry) if isinstance(self.origin, GrpcServerRepositoryLocationOrigin): self._port = self.origin.port self._socket = self.origin.socket self._host = self.origin.host self._use_ssl = bool(self.origin.use_ssl) else: self._port = check.opt_int_param(port, "port") self._socket = check.opt_str_param(socket, "socket") self._host = check.str_param(host, "host") self._use_ssl = False self._watch_thread_shutdown_event = None self._watch_thread = None self._heartbeat_shutdown_event = None self._heartbeat_thread = None self._heartbeat = check.bool_param(heartbeat, "heartbeat") self._watch_server = check.bool_param(watch_server, "watch_server") self.server_id = None self._external_repositories_data = None self._executable_path = None self._container_image = None self._repository_code_pointer_dict = None try: self.client = DagsterGrpcClient( port=self._port, socket=self._socket, host=self._host, use_ssl=self._use_ssl, ) list_repositories_response = sync_list_repositories_grpc( self.client) self.server_id = server_id if server_id else sync_get_server_id( self.client) self.repository_names = set( symbol.repository_name for symbol in list_repositories_response.repository_symbols) if self._heartbeat: self._heartbeat_shutdown_event = threading.Event() self._heartbeat_thread = threading.Thread( target=client_heartbeat_thread, args=( self.client, self._heartbeat_shutdown_event, ), name="grpc-client-heartbeat", ) self._heartbeat_thread.daemon = True self._heartbeat_thread.start() self._executable_path = list_repositories_response.executable_path self._repository_code_pointer_dict = ( list_repositories_response.repository_code_pointer_dict) self._container_image = self._reload_current_image() self._external_repositories_data = sync_get_streaming_external_repositories_data_grpc( self.client, self, ) self.external_repositories = { repo_name: ExternalRepository( repo_data, RepositoryHandle( repository_name=repo_name, repository_location=self, ), ) for repo_name, repo_data in self._external_repositories_data.items() } except: self.cleanup() raise
def __init__( self, origin: RepositoryLocationOrigin, host: Optional[str] = None, port: Optional[int] = None, socket: Optional[str] = None, server_id: Optional[str] = None, heartbeat: Optional[bool] = False, watch_server: Optional[bool] = True, grpc_server_registry: Optional[GrpcServerRegistry] = None, ): from dagster.grpc.client import DagsterGrpcClient, client_heartbeat_thread from dagster.grpc.server_watcher import create_grpc_watch_thread self._origin = check.inst_param(origin, "origin", RepositoryLocationOrigin) self.grpc_server_registry = check.opt_inst_param( grpc_server_registry, "grpc_server_registry", GrpcServerRegistry ) if isinstance(self.origin, GrpcServerRepositoryLocationOrigin): self._port = self.origin.port self._socket = self.origin.socket self._host = self.origin.host self._use_ssl = bool(self.origin.use_ssl) else: self._port = check.opt_int_param(port, "port") self._socket = check.opt_str_param(socket, "socket") self._host = check.str_param(host, "host") self._use_ssl = False self._watch_thread_shutdown_event = None self._watch_thread = None self._heartbeat_shutdown_event = None self._heartbeat_thread = None self._heartbeat = check.bool_param(heartbeat, "heartbeat") self._watch_server = check.bool_param(watch_server, "watch_server") self.server_id = None self._external_repositories_data = None self._executable_path = None self._container_image = None self._repository_code_pointer_dict = None try: self.client = DagsterGrpcClient( port=self._port, socket=self._socket, host=self._host, use_ssl=self._use_ssl, ) list_repositories_response = sync_list_repositories_grpc(self.client) self.server_id = server_id if server_id else sync_get_server_id(self.client) self.repository_names = set( symbol.repository_name for symbol in list_repositories_response.repository_symbols ) if self._heartbeat: self._heartbeat_shutdown_event = threading.Event() self._heartbeat_thread = threading.Thread( target=client_heartbeat_thread, args=( self.client, self._heartbeat_shutdown_event, ), name="grpc-client-heartbeat", ) self._heartbeat_thread.daemon = True self._heartbeat_thread.start() if self._watch_server: self._state_subscribers: List[LocationStateSubscriber] = [] self._watch_thread_shutdown_event, self._watch_thread = create_grpc_watch_thread( self.client, on_updated=lambda new_server_id: self._send_state_event_to_subscribers( LocationStateChangeEvent( LocationStateChangeEventType.LOCATION_UPDATED, location_name=self.name, message="Server has been updated.", server_id=new_server_id, ) ), on_error=lambda: self._send_state_event_to_subscribers( LocationStateChangeEvent( LocationStateChangeEventType.LOCATION_ERROR, location_name=self.name, message="Unable to reconnect to server. You can reload the server once it is " "reachable again", ) ), ) self._watch_thread.start() self._executable_path = list_repositories_response.executable_path self._repository_code_pointer_dict = ( list_repositories_response.repository_code_pointer_dict ) self._container_image = self._reload_current_image() self._external_repositories_data = sync_get_streaming_external_repositories_data_grpc( self.client, self, ) self.external_repositories = { repo_name: ExternalRepository( repo_data, RepositoryHandle( repository_name=repo_name, repository_location=self, ), ) for repo_name, repo_data in self._external_repositories_data.items() } except: self.cleanup() raise