def grpc_schedule_origin(schedule_name): loadable_target_origin = LoadableTargetOrigin( executable_path=sys.executable, python_file=__file__, attribute="the_repo" ) server_process = GrpcServerProcess(loadable_target_origin=loadable_target_origin) with server_process.create_ephemeral_client() as api_client: repo_origin = RepositoryGrpcServerOrigin( host=api_client.host, port=api_client.port, socket=api_client.socket, repository_name="the_repo", ) yield repo_origin.get_schedule_origin(schedule_name) server_process.wait()
def get_origin(self): if isinstance(self.repository_location_handle, InProcessRepositoryLocationHandle): return RepositoryPythonOrigin( code_pointer=self.repository_location_handle. repository_code_pointer_dict[self.repository_name], executable_path=sys.executable, ) elif isinstance(self.repository_location_handle, ManagedGrpcPythonEnvRepositoryLocationHandle): return RepositoryPythonOrigin( code_pointer=self.repository_location_handle. repository_code_pointer_dict[self.repository_name], executable_path=self.repository_location_handle. executable_path, ) elif isinstance(self.repository_location_handle, GrpcServerRepositoryLocationHandle): return RepositoryGrpcServerOrigin( host=self.repository_location_handle.host, port=self.repository_location_handle.port, socket=self.repository_location_handle.socket, repository_name=self.repository_name, ) else: check.failed( "Can not target represented RepositoryDefinition locally for repository from a {}." .format(self.repository_location_handle.__class__.__name__))
def sync_get_external_repositories_grpc(api_client, repository_location_handle): check.inst_param(repository_location_handle, "repository_location_handle", RepositoryLocationHandle) repos = [] for repository_name in repository_location_handle.repository_names: external_repository_data = check.inst( api_client.external_repository( repository_grpc_server_origin=RepositoryGrpcServerOrigin( repository_location_handle.host, repository_location_handle.port, repository_location_handle.socket, repository_name, )), ExternalRepositoryData, ) repos.append( ExternalRepository( external_repository_data, RepositoryHandle( repository_name=external_repository_data.name, repository_location_handle=repository_location_handle, ), )) return repos
def sync_get_streaming_external_repositories_grpc(api_client, repository_location_handle): check.inst_param(repository_location_handle, "repository_location_handle", RepositoryLocationHandle) repos = [] for repository_name in repository_location_handle.repository_names: external_repository_chunks = list( api_client.streaming_external_repository( repository_grpc_server_origin=RepositoryGrpcServerOrigin( repository_location_handle.host, repository_location_handle.port, repository_location_handle.socket, repository_name, ))) external_repository_data = deserialize_json_to_dagster_namedtuple( "".join([ chunk["serialized_external_repository_chunk"] for chunk in external_repository_chunks ])) repos.append( ExternalRepository( external_repository_data, RepositoryHandle( repository_name=external_repository_data.name, repository_location_handle=repository_location_handle, ), )) return repos
def get_repository_origin_from_kwargs(kwargs): load_target = created_workspace_load_target(kwargs) if isinstance(load_target, PythonFileTarget): return RepositoryPythonOrigin( executable_path=sys.executable, code_pointer=CodePointer.from_python_file( load_target.python_file, load_target.attribute, working_directory=load_target.working_directory, ), ) elif isinstance(load_target, ModuleTarget): return RepositoryPythonOrigin( executable_path=sys.executable, code_pointer=CodePointer.from_module(load_target.module_name, load_target.attribute), ) elif isinstance(load_target, GrpcServerTarget): return RepositoryGrpcServerOrigin( host=load_target.host, port=load_target.port, socket=load_target.socket, repository_name=kwargs['repository'], ) else: check.failed('invalid')
def test_grpc_server_down(): with _default_instance() as instance: down_grpc_repo_origin = RepositoryGrpcServerOrigin( host="localhost", port=find_free_port(), socket=None, repository_name="down_repo" ) down_grpc_schedule_origin = down_grpc_repo_origin.get_schedule_origin("down_schedule") instance = DagsterInstance.get() result = sync_launch_scheduled_execution(down_grpc_schedule_origin) assert isinstance(result, ScheduledExecutionFailed) assert "failed to connect to all addresses" in result.errors[0].to_string() ticks = instance.get_schedule_ticks(down_grpc_schedule_origin.get_id()) assert ticks[0].status == ScheduleTickStatus.FAILURE assert "failed to connect to all addresses" in ticks[0].error.message
def grpc_schedule_origin(schedule_name): with seven.TemporaryDirectory() as temp_dir: with environ({'DAGSTER_HOME': temp_dir}): loadable_target_origin = LoadableTargetOrigin( executable_path=sys.executable, python_file=__file__, attribute='the_repo') with GrpcServerProcess( loadable_target_origin=loadable_target_origin ).create_ephemeral_client() as api_client: repo_origin = RepositoryGrpcServerOrigin( host=api_client.host, port=api_client.port, socket=api_client.socket, repository_name='the_repo', ) yield repo_origin.get_schedule_origin(schedule_name)
def test_cancel_run(): with instance_for_test() as instance: loadable_target_origin = LoadableTargetOrigin( executable_path=sys.executable, python_file=__file__, working_directory=None, ) server_process = GrpcServerProcess(loadable_target_origin, max_workers=10) with server_process.create_ephemeral_client() as api_client: streaming_results = [] pipeline_run = instance.create_run_for_pipeline( streaming_pipeline, run_config={"solids": {"streamer": {"config": {"length": 20}}}}, ) execute_run_args = ExecuteRunArgs( pipeline_origin=PipelineGrpcServerOrigin( pipeline_name="streaming_pipeline", repository_origin=RepositoryGrpcServerOrigin( host="localhost", socket=api_client.socket, port=api_client.port, repository_name="test_repository", ), ), pipeline_run_id=pipeline_run.run_id, instance_ref=instance.get_ref(), ) stream_events_result_thread = threading.Thread( target=_stream_events_target, args=[streaming_results, api_client, execute_run_args] ) stream_events_result_thread.daemon = True stream_events_result_thread.start() poll_for_step_start(instance, pipeline_run.run_id) res = api_client.cancel_execution( cancel_execution_request=CancelExecutionRequest(run_id=pipeline_run.run_id) ) assert res.success is True poll_for_run(instance, pipeline_run.run_id) logs = instance.all_logs(pipeline_run.run_id) assert ( len( [ ev for ev in logs if ev.dagster_event.event_type_value == "STEP_MATERIALIZATION" ] ) < 20 ) # soft termination assert [ev for ev in logs if ev.dagster_event.event_type_value == "STEP_FAILURE"] server_process.wait()
def get_repository_origin_from_kwargs(kwargs): load_target = created_workspace_load_target(kwargs) if isinstance(load_target, GrpcServerTarget): return RepositoryGrpcServerOrigin( host=load_target.host, port=load_target.port, socket=load_target.socket, repository_name=kwargs["repository"], ) return get_repository_python_origin_from_kwargs(kwargs)
def test_grpc_server_down(): with seven.TemporaryDirectory() as temp_dir: with environ({'DAGSTER_HOME': temp_dir}): down_grpc_repo_origin = RepositoryGrpcServerOrigin( host='localhost', port=find_free_port(), socket=None, repository_name='down_repo') down_grpc_schedule_origin = down_grpc_repo_origin.get_schedule_origin( 'down_schedule') instance = DagsterInstance.get() result = sync_launch_scheduled_execution(down_grpc_schedule_origin) assert isinstance(result, ScheduledExecutionFailed) assert 'failed to connect to all addresses' in result.errors[ 0].to_string() ticks = instance.get_schedule_ticks( down_grpc_schedule_origin.get_id()) assert ticks[0].status == ScheduleTickStatus.FAILURE assert 'failed to connect to all addresses' in ticks[ 0].error.message