def get_external_pipeline_or_job_from_external_repo( external_repo: ExternalRepository, provided_pipeline_or_job_name: Optional[str], using_job_op_graph_apis: bool = False, ) -> ExternalPipeline: check.inst_param(external_repo, "external_repo", ExternalRepository) check.opt_str_param(provided_pipeline_or_job_name, "provided_pipeline_or_job_name") external_pipelines = { ep.name: ep for ep in (external_repo.get_external_jobs() if using_job_op_graph_apis else external_repo.get_all_external_pipelines()) } check.invariant(external_pipelines) if provided_pipeline_or_job_name is None and len(external_pipelines) == 1: return next(iter(external_pipelines.values())) if provided_pipeline_or_job_name is None: raise click.UsageError( ("Must provide {flag} as there is more than one pipeline/job " "in {repository}. Options are: {pipelines}.").format( flag="--job" if using_job_op_graph_apis else "--pipeline", repository=external_repo.name, pipelines=_sorted_quoted(external_pipelines.keys()), )) if not provided_pipeline_or_job_name in external_pipelines: raise click.UsageError(( '{pipeline_or_job} "{provided_pipeline_name}" not found in repository "{repository_name}". ' "Found {found_names} instead.").format( pipeline_or_job="Job" if using_job_op_graph_apis else "Pipeline", provided_pipeline_name=provided_pipeline_or_job_name, repository_name=external_repo.name, found_names=_sorted_quoted(external_pipelines.keys()), )) return external_pipelines[provided_pipeline_or_job_name]
def create_external_repositories(self): from dagster.core.host_representation.external import ExternalRepository return { repo_name: ExternalRepository( repo_data, RepositoryHandle( repository_name=repo_name, repository_location_handle=self, ), ) for repo_name, repo_data in self._external_repositories_data.items() }
def __init__( self, origin: RepositoryLocationOrigin, host: Optional[str] = None, port: Optional[int] = None, socket: Optional[str] = None, server_id: Optional[str] = None, heartbeat: Optional[bool] = False, watch_server: Optional[bool] = True, grpc_server_registry: Optional[GrpcServerRegistry] = None, ): from dagster.grpc.client import DagsterGrpcClient, client_heartbeat_thread self._origin = check.inst_param(origin, "origin", RepositoryLocationOrigin) self.grpc_server_registry = check.opt_inst_param( grpc_server_registry, "grpc_server_registry", GrpcServerRegistry) if isinstance(self.origin, GrpcServerRepositoryLocationOrigin): self._port = self.origin.port self._socket = self.origin.socket self._host = self.origin.host self._use_ssl = bool(self.origin.use_ssl) else: self._port = check.opt_int_param(port, "port") self._socket = check.opt_str_param(socket, "socket") self._host = check.str_param(host, "host") self._use_ssl = False self._watch_thread_shutdown_event = None self._watch_thread = None self._heartbeat_shutdown_event = None self._heartbeat_thread = None self._heartbeat = check.bool_param(heartbeat, "heartbeat") self._watch_server = check.bool_param(watch_server, "watch_server") self.server_id = None self._external_repositories_data = None self._executable_path = None self._container_image = None self._repository_code_pointer_dict = None try: self.client = DagsterGrpcClient( port=self._port, socket=self._socket, host=self._host, use_ssl=self._use_ssl, ) list_repositories_response = sync_list_repositories_grpc( self.client) self.server_id = server_id if server_id else sync_get_server_id( self.client) self.repository_names = set( symbol.repository_name for symbol in list_repositories_response.repository_symbols) if self._heartbeat: self._heartbeat_shutdown_event = threading.Event() self._heartbeat_thread = threading.Thread( target=client_heartbeat_thread, args=( self.client, self._heartbeat_shutdown_event, ), name="grpc-client-heartbeat", ) self._heartbeat_thread.daemon = True self._heartbeat_thread.start() self._executable_path = list_repositories_response.executable_path self._repository_code_pointer_dict = ( list_repositories_response.repository_code_pointer_dict) self._container_image = self._reload_current_image() self._external_repositories_data = sync_get_streaming_external_repositories_data_grpc( self.client, self, ) self.external_repositories = { repo_name: ExternalRepository( repo_data, RepositoryHandle( repository_name=repo_name, repository_location=self, ), ) for repo_name, repo_data in self._external_repositories_data.items() } except: self.cleanup() raise
def __init__( self, origin: RepositoryLocationOrigin, host: Optional[str] = None, port: Optional[int] = None, socket: Optional[str] = None, server_id: Optional[str] = None, heartbeat: Optional[bool] = False, watch_server: Optional[bool] = True, grpc_server_registry: Optional[GrpcServerRegistry] = None, ): from dagster.grpc.client import DagsterGrpcClient, client_heartbeat_thread from dagster.grpc.server_watcher import create_grpc_watch_thread self._origin = check.inst_param(origin, "origin", RepositoryLocationOrigin) self.grpc_server_registry = check.opt_inst_param( grpc_server_registry, "grpc_server_registry", GrpcServerRegistry ) if isinstance(self.origin, GrpcServerRepositoryLocationOrigin): self._port = self.origin.port self._socket = self.origin.socket self._host = self.origin.host self._use_ssl = bool(self.origin.use_ssl) else: self._port = check.opt_int_param(port, "port") self._socket = check.opt_str_param(socket, "socket") self._host = check.str_param(host, "host") self._use_ssl = False self._watch_thread_shutdown_event = None self._watch_thread = None self._heartbeat_shutdown_event = None self._heartbeat_thread = None self._heartbeat = check.bool_param(heartbeat, "heartbeat") self._watch_server = check.bool_param(watch_server, "watch_server") self.server_id = None self._external_repositories_data = None self._executable_path = None self._container_image = None self._repository_code_pointer_dict = None try: self.client = DagsterGrpcClient( port=self._port, socket=self._socket, host=self._host, use_ssl=self._use_ssl, ) list_repositories_response = sync_list_repositories_grpc(self.client) self.server_id = server_id if server_id else sync_get_server_id(self.client) self.repository_names = set( symbol.repository_name for symbol in list_repositories_response.repository_symbols ) if self._heartbeat: self._heartbeat_shutdown_event = threading.Event() self._heartbeat_thread = threading.Thread( target=client_heartbeat_thread, args=( self.client, self._heartbeat_shutdown_event, ), name="grpc-client-heartbeat", ) self._heartbeat_thread.daemon = True self._heartbeat_thread.start() if self._watch_server: self._state_subscribers: List[LocationStateSubscriber] = [] self._watch_thread_shutdown_event, self._watch_thread = create_grpc_watch_thread( self.client, on_updated=lambda new_server_id: self._send_state_event_to_subscribers( LocationStateChangeEvent( LocationStateChangeEventType.LOCATION_UPDATED, location_name=self.name, message="Server has been updated.", server_id=new_server_id, ) ), on_error=lambda: self._send_state_event_to_subscribers( LocationStateChangeEvent( LocationStateChangeEventType.LOCATION_ERROR, location_name=self.name, message="Unable to reconnect to server. You can reload the server once it is " "reachable again", ) ), ) self._watch_thread.start() self._executable_path = list_repositories_response.executable_path self._repository_code_pointer_dict = ( list_repositories_response.repository_code_pointer_dict ) self._container_image = self._reload_current_image() self._external_repositories_data = sync_get_streaming_external_repositories_data_grpc( self.client, self, ) self.external_repositories = { repo_name: ExternalRepository( repo_data, RepositoryHandle( repository_name=repo_name, repository_location=self, ), ) for repo_name, repo_data in self._external_repositories_data.items() } except: self.cleanup() raise