def test_execute_run_api_grpc_python_handle(pipeline_handle): with seven.TemporaryDirectory() as temp_dir: instance = DagsterInstance.local_temp(temp_dir) pipeline_run = instance.create_run( pipeline_name='foo', run_id=None, run_config={}, mode='default', solids_to_execute=None, step_keys_to_execute=None, status=None, tags=None, root_run_id=None, parent_run_id=None, pipeline_snapshot=None, execution_plan_snapshot=None, parent_pipeline_snapshot=None, ) loadable_target_origin = LoadableTargetOrigin.from_python_origin( pipeline_handle.get_origin().repository_origin) with GrpcServerProcess(loadable_target_origin, max_workers=2) as server_process: api_client = server_process.create_ephemeral_client() events = [ event for event in sync_execute_run_grpc( api_client=api_client, instance_ref=instance.get_ref(), pipeline_origin=pipeline_handle.get_origin(), pipeline_run=pipeline_run, ) ] assert len(events) == 14 assert [event.event_type_value for event in events] == [ 'ENGINE_EVENT', 'ENGINE_EVENT', 'PIPELINE_START', 'ENGINE_EVENT', 'STEP_START', 'STEP_OUTPUT', 'STEP_SUCCESS', 'STEP_START', 'STEP_INPUT', 'STEP_OUTPUT', 'STEP_SUCCESS', 'ENGINE_EVENT', 'PIPELINE_SUCCESS', 'ENGINE_EVENT', ]
def get_external_pipeline_from_grpc_server_repository(pipeline_name): repo_yaml = file_relative_path(__file__, 'repo.yaml') recon_repo = ReconstructableRepository.from_legacy_repository_yaml(repo_yaml) loadable_target_origin = LoadableTargetOrigin.from_python_origin(recon_repo.get_origin()) with GrpcServerProcess( loadable_target_origin=loadable_target_origin ).create_ephemeral_client() as server: repository_location = GrpcServerRepositoryLocation( RepositoryLocationHandle.create_grpc_server_location( location_name='test', port=server.port, socket=server.socket, host='localhost', ) ) yield repository_location.get_repository('nope').get_full_external_pipeline(pipeline_name)
def _mgr_fn(recon_repo): '''Goes out of process via grpc''' check.inst_param(recon_repo, 'recon_repo', ReconstructableRepository) loadable_target_origin = LoadableTargetOrigin.from_python_origin( recon_repo.get_origin() ) yield [ GrpcServerRepositoryLocation( RepositoryLocationHandle.create_process_bound_grpc_server_location( loadable_target_origin=loadable_target_origin, location_name='test', ) ) ]
def test_run_always_finishes(temp_instance): # pylint: disable=redefined-outer-name instance = temp_instance pipeline_run = instance.create_run_for_pipeline(pipeline_def=slow_pipeline, run_config=None) run_id = pipeline_run.run_id recon_repo = ReconstructableRepository.for_file(__file__, 'nope') loadable_target_origin = LoadableTargetOrigin.from_python_origin(recon_repo.get_origin()) server_process = GrpcServerProcess(loadable_target_origin=loadable_target_origin, max_workers=4) with server_process.create_ephemeral_client() as api_client: repository_location = GrpcServerRepositoryLocation( RepositoryLocationHandle.create_grpc_server_location( location_name='test', port=api_client.port, socket=api_client.socket, host=api_client.host, ) ) external_pipeline = repository_location.get_repository('nope').get_full_external_pipeline( 'slow_pipeline' ) assert instance.get_run_by_id(run_id).status == PipelineRunStatus.NOT_STARTED launcher = instance.run_launcher launcher.launch_run( instance=instance, run=pipeline_run, external_pipeline=external_pipeline ) # Server process now receives shutdown event, run has not finished yet pipeline_run = instance.get_run_by_id(run_id) assert not pipeline_run.is_finished assert server_process.server_process.poll() is None # Server should wait until run finishes, then shutdown pipeline_run = poll_for_run(instance, run_id) assert pipeline_run.status == PipelineRunStatus.SUCCESS start_time = time.time() while server_process.server_process.poll() is None: time.sleep(0.05) # Verify server process cleans up eventually assert time.time() - start_time < 5
def _mgr_fn(recon_repo): check.inst_param(recon_repo, 'recon_repo', ReconstructableRepository) loadable_target_origin = LoadableTargetOrigin.from_python_origin( recon_repo.get_origin() ) with GrpcServerProcess(loadable_target_origin=loadable_target_origin) as server: yield [ GrpcServerRepositoryLocation( RepositoryLocationHandle.create_grpc_server_location( port=server.port, socket=server.socket, host='localhost', location_name='test', ) ) ]
def _ephemeral_launched_run_client( instance_ref, pipeline_origin, pipeline_run_id, cancellation_event ): '''Spins up an ephemeral client & server with two workers. This is to allow for cancellation to be processed as an interrupt rather than waiting for the launched run to complete.''' check.inst_param(instance_ref, 'instance_ref', InstanceRef) check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin) check.str_param(pipeline_run_id, 'pipeline_run_id') check.inst_param(cancellation_event, 'cancellation_event', multiprocessing.synchronize.Event) instance = DagsterInstance.from_ref(instance_ref) pipeline_run = instance.get_run_by_id(pipeline_run_id) loadable_target_origin = LoadableTargetOrigin.from_python_origin( pipeline_origin.repository_origin ) with GrpcServerProcess(loadable_target_origin, max_workers=2) as server_process: api_client = server_process.create_ephemeral_client() execute_run_thread = threading.Thread( target=sync_execute_run_grpc, kwargs={ 'api_client': api_client, 'instance_ref': instance_ref, 'pipeline_origin': pipeline_origin, 'pipeline_run': pipeline_run, }, ) execute_run_thread.start() while execute_run_thread.is_alive(): if cancellation_event.is_set(): api_client.cancel_execution(CancelExecutionRequest(run_id=pipeline_run_id)) execute_run_thread.join() time.sleep(SUBPROCESS_TICK)