def _start_watch_thread( self, origin: GrpcServerRepositoryLocationOrigin) -> None: location_name = origin.location_name check.invariant( location_name not in self._watch_thread_shutdown_events) client = origin.create_client() shutdown_event, watch_thread = create_grpc_watch_thread( location_name, client, on_updated=lambda location_name, new_server_id: self. _send_state_event_to_subscribers( LocationStateChangeEvent( LocationStateChangeEventType.LOCATION_UPDATED, location_name=location_name, message="Server has been updated.", server_id=new_server_id, )), on_error=lambda location_name: self. _send_state_event_to_subscribers( LocationStateChangeEvent( LocationStateChangeEventType.LOCATION_ERROR, location_name=location_name, message= "Unable to reconnect to server. You can reload the server once it is " "reachable again", )), ) self._watch_thread_shutdown_events[location_name] = shutdown_event self._watch_threads[location_name] = watch_thread watch_thread.start()
def _location_origin_from_grpc_server_config( grpc_server_config: Dict, yaml_path: str) -> GrpcServerRepositoryLocationOrigin: check.dict_param(grpc_server_config, "grpc_server_config") check.str_param(yaml_path, "yaml_path") port, socket, host, location_name, use_ssl = ( grpc_server_config.get("port"), grpc_server_config.get("socket"), grpc_server_config.get("host"), grpc_server_config.get("location_name"), grpc_server_config.get("ssl"), ) check.invariant((socket or port) and not (socket and port), "must supply either a socket or a port") if not host: host = "localhost" return GrpcServerRepositoryLocationOrigin( port=port, socket=socket, host=host, location_name=location_name, use_ssl=use_ssl, )
def test_sensor_timeout(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") subprocess_args = [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, ] process = subprocess.Popen( subprocess_args, stdout=subprocess.PIPE, ) try: wait_for_grpc_server( process, DagsterGrpcClient(port=port, host="localhost"), subprocess_args ) client = DagsterGrpcClient(port=port) with instance_for_test() as instance: repo_origin = ExternalRepositoryOrigin( repository_location_origin=GrpcServerRepositoryLocationOrigin( port=port, host="localhost" ), repository_name="bar_repo", ) with pytest.raises(DagsterUserCodeUnreachableError) as exc_info: client.external_sensor_execution( sensor_execution_args=SensorExecutionArgs( repository_origin=repo_origin, instance_ref=instance.get_ref(), sensor_name="slow_sensor", last_completion_time=None, last_run_key=None, cursor=None, ), timeout=2, ) assert "Deadline Exceeded" in str(exc_info.getrepr()) # Call succeeds without the timeout client.external_sensor_execution( sensor_execution_args=SensorExecutionArgs( repository_origin=repo_origin, instance_ref=instance.get_ref(), sensor_name="slow_sensor", last_completion_time=None, last_run_key=None, cursor=None, ), ) finally: process.terminate()
def create_origins(self): return [ GrpcServerRepositoryLocationOrigin( port=self.port, socket=self.socket, host=self.host, location_name=self.location_name, ) ]
def test_sensor_timeout(): port = find_free_port() python_file = file_relative_path(__file__, "grpc_repo.py") ipc_output_file = _get_ipc_output_file() process = subprocess.Popen( [ "dagster", "api", "grpc", "--port", str(port), "--python-file", python_file, "--ipc-output-file", ipc_output_file, ], stdout=subprocess.PIPE, ) try: wait_for_grpc_server(process, ipc_output_file) client = DagsterGrpcClient(port=port) with instance_for_test() as instance: repo_origin = ExternalRepositoryOrigin( repository_location_origin=GrpcServerRepositoryLocationOrigin( port=port, host="localhost" ), repository_name="bar_repo", ) with pytest.raises(Exception, match="Deadline Exceeded"): client.external_sensor_execution( sensor_execution_args=SensorExecutionArgs( repository_origin=repo_origin, instance_ref=instance.get_ref(), sensor_name="slow_sensor", last_completion_time=None, last_run_key=None, ), timeout=2, ) # Call succeeds without the timeout client.external_sensor_execution( sensor_execution_args=SensorExecutionArgs( repository_origin=repo_origin, instance_ref=instance.get_ref(), sensor_name="slow_sensor", last_completion_time=None, last_run_key=None, ), ) finally: process.terminate()
def create_from_repository_origin(repository_origin, instance): check.inst_param(repository_origin, "repository_origin", RepositoryOrigin) check.inst_param(instance, "instance", DagsterInstance) if isinstance(repository_origin, RepositoryGrpcServerOrigin): return RepositoryLocationHandle.create_from_repository_location_origin( GrpcServerRepositoryLocationOrigin( port=repository_origin.port, socket=repository_origin.socket, host=repository_origin.host, )) elif isinstance(repository_origin, RepositoryPythonOrigin): loadable_target_origin = repository_origin.loadable_target_origin repo_location_origin = ManagedGrpcPythonEnvRepositoryLocationOrigin( loadable_target_origin) return RepositoryLocationHandle.create_from_repository_location_origin( repo_location_origin) else: raise DagsterInvariantViolationError( "Unexpected repository origin type")
def test_external_job_origin_instigator_origin(): def build_legacy_whitelist_map(): legacy_env = WhitelistMap.create() @_whitelist_for_serdes(legacy_env) class ExternalJobOrigin( namedtuple("_ExternalJobOrigin", "external_repository_origin job_name")): def get_id(self): return create_snapshot_id(self) @_whitelist_for_serdes(legacy_env) class ExternalRepositoryOrigin( namedtuple("_ExternalRepositoryOrigin", "repository_location_origin repository_name")): def get_id(self): return create_snapshot_id(self) class GrpcServerOriginSerializer(DefaultNamedTupleSerializer): @classmethod def skip_when_empty(cls): return {"use_ssl"} @_whitelist_for_serdes(whitelist_map=legacy_env, serializer=GrpcServerOriginSerializer) class GrpcServerRepositoryLocationOrigin( namedtuple( "_GrpcServerRepositoryLocationOrigin", "host port socket location_name use_ssl", ), ): def __new__(cls, host, port=None, socket=None, location_name=None, use_ssl=None): return super(GrpcServerRepositoryLocationOrigin, cls).__new__(cls, host, port, socket, location_name, use_ssl) return ( legacy_env, ExternalJobOrigin, ExternalRepositoryOrigin, GrpcServerRepositoryLocationOrigin, ) legacy_env, klass, repo_klass, location_klass = build_legacy_whitelist_map( ) from dagster.core.host_representation.origin import ( ExternalInstigatorOrigin, ExternalRepositoryOrigin, GrpcServerRepositoryLocationOrigin, ) # serialize from current code, compare against old code instigator_origin = ExternalInstigatorOrigin( external_repository_origin=ExternalRepositoryOrigin( repository_location_origin=GrpcServerRepositoryLocationOrigin( host="localhost", port=1234, location_name="test_location"), repository_name="the_repo", ), instigator_name="simple_schedule", ) instigator_origin_str = serialize_dagster_namedtuple(instigator_origin) instigator_to_job = _deserialize_json(instigator_origin_str, legacy_env) assert isinstance(instigator_to_job, klass) # ensure that the origin id is stable assert instigator_to_job.get_id() == instigator_origin.get_id() # # serialize from old code, compare against current code job_origin = klass( external_repository_origin=repo_klass( repository_location_origin=location_klass( host="localhost", port=1234, location_name="test_location"), repository_name="the_repo", ), job_name="simple_schedule", ) job_origin_str = serialize_value(job_origin, legacy_env) from dagster.serdes.serdes import _WHITELIST_MAP job_to_instigator = deserialize_json_to_dagster_namedtuple(job_origin_str) assert isinstance(job_to_instigator, ExternalInstigatorOrigin) # ensure that the origin id is stable assert job_to_instigator.get_id() == job_origin.get_id()