def grpc_schedule_origin(schedule_name):
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable, python_file=__file__, attribute="the_repo"
)
server_process = GrpcServerProcess(loadable_target_origin=loadable_target_origin)
with server_process.create_ephemeral_client() as api_client:
repo_origin = RepositoryGrpcServerOrigin(
host=api_client.host,
port=api_client.port,
socket=api_client.socket,
repository_name="the_repo",
)
yield repo_origin.get_schedule_origin(schedule_name)
server_process.wait()
def get_origin(self):
if isinstance(self.repository_location_handle,
InProcessRepositoryLocationHandle):
return RepositoryPythonOrigin(
code_pointer=self.repository_location_handle.
repository_code_pointer_dict[self.repository_name],
executable_path=sys.executable,
)
elif isinstance(self.repository_location_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle):
return RepositoryPythonOrigin(
code_pointer=self.repository_location_handle.
repository_code_pointer_dict[self.repository_name],
executable_path=self.repository_location_handle.
executable_path,
)
elif isinstance(self.repository_location_handle,
GrpcServerRepositoryLocationHandle):
return RepositoryGrpcServerOrigin(
host=self.repository_location_handle.host,
port=self.repository_location_handle.port,
socket=self.repository_location_handle.socket,
repository_name=self.repository_name,
)
else:
check.failed(
"Can not target represented RepositoryDefinition locally for repository from a {}."
.format(self.repository_location_handle.__class__.__name__))
def sync_get_external_repositories_grpc(api_client,
repository_location_handle):
check.inst_param(repository_location_handle, "repository_location_handle",
RepositoryLocationHandle)
repos = []
for repository_name in repository_location_handle.repository_names:
external_repository_data = check.inst(
api_client.external_repository(
repository_grpc_server_origin=RepositoryGrpcServerOrigin(
repository_location_handle.host,
repository_location_handle.port,
repository_location_handle.socket,
repository_name,
)),
ExternalRepositoryData,
)
repos.append(
ExternalRepository(
external_repository_data,
RepositoryHandle(
repository_name=external_repository_data.name,
repository_location_handle=repository_location_handle,
),
))
return repos
def sync_get_streaming_external_repositories_grpc(api_client,
repository_location_handle):
check.inst_param(repository_location_handle, "repository_location_handle",
RepositoryLocationHandle)
repos = []
for repository_name in repository_location_handle.repository_names:
external_repository_chunks = list(
api_client.streaming_external_repository(
repository_grpc_server_origin=RepositoryGrpcServerOrigin(
repository_location_handle.host,
repository_location_handle.port,
repository_location_handle.socket,
repository_name,
)))
external_repository_data = deserialize_json_to_dagster_namedtuple(
"".join([
chunk["serialized_external_repository_chunk"]
for chunk in external_repository_chunks
]))
repos.append(
ExternalRepository(
external_repository_data,
RepositoryHandle(
repository_name=external_repository_data.name,
repository_location_handle=repository_location_handle,
),
))
return repos
def get_repository_origin_from_kwargs(kwargs):
load_target = created_workspace_load_target(kwargs)
if isinstance(load_target, PythonFileTarget):
return RepositoryPythonOrigin(
executable_path=sys.executable,
code_pointer=CodePointer.from_python_file(
load_target.python_file,
load_target.attribute,
working_directory=load_target.working_directory,
),
)
elif isinstance(load_target, ModuleTarget):
return RepositoryPythonOrigin(
executable_path=sys.executable,
code_pointer=CodePointer.from_module(load_target.module_name,
load_target.attribute),
)
elif isinstance(load_target, GrpcServerTarget):
return RepositoryGrpcServerOrigin(
host=load_target.host,
port=load_target.port,
socket=load_target.socket,
repository_name=kwargs['repository'],
)
else:
check.failed('invalid')
def test_grpc_server_down():
with _default_instance() as instance:
down_grpc_repo_origin = RepositoryGrpcServerOrigin(
host="localhost", port=find_free_port(), socket=None, repository_name="down_repo"
)
down_grpc_schedule_origin = down_grpc_repo_origin.get_schedule_origin("down_schedule")
instance = DagsterInstance.get()
result = sync_launch_scheduled_execution(down_grpc_schedule_origin)
assert isinstance(result, ScheduledExecutionFailed)
assert "failed to connect to all addresses" in result.errors[0].to_string()
ticks = instance.get_schedule_ticks(down_grpc_schedule_origin.get_id())
assert ticks[0].status == ScheduleTickStatus.FAILURE
assert "failed to connect to all addresses" in ticks[0].error.message
def grpc_schedule_origin(schedule_name):
with seven.TemporaryDirectory() as temp_dir:
with environ({'DAGSTER_HOME': temp_dir}):
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=__file__,
attribute='the_repo')
with GrpcServerProcess(
loadable_target_origin=loadable_target_origin
).create_ephemeral_client() as api_client:
repo_origin = RepositoryGrpcServerOrigin(
host=api_client.host,
port=api_client.port,
socket=api_client.socket,
repository_name='the_repo',
)
yield repo_origin.get_schedule_origin(schedule_name)
def test_cancel_run():
with instance_for_test() as instance:
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable, python_file=__file__, working_directory=None,
)
server_process = GrpcServerProcess(loadable_target_origin, max_workers=10)
with server_process.create_ephemeral_client() as api_client:
streaming_results = []
pipeline_run = instance.create_run_for_pipeline(
streaming_pipeline, run_config={"solids": {"streamer": {"config": {"length": 20}}}},
)
execute_run_args = ExecuteRunArgs(
pipeline_origin=PipelineGrpcServerOrigin(
pipeline_name="streaming_pipeline",
repository_origin=RepositoryGrpcServerOrigin(
host="localhost",
socket=api_client.socket,
port=api_client.port,
repository_name="test_repository",
),
),
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance.get_ref(),
)
stream_events_result_thread = threading.Thread(
target=_stream_events_target, args=[streaming_results, api_client, execute_run_args]
)
stream_events_result_thread.daemon = True
stream_events_result_thread.start()
poll_for_step_start(instance, pipeline_run.run_id)
res = api_client.cancel_execution(
cancel_execution_request=CancelExecutionRequest(run_id=pipeline_run.run_id)
)
assert res.success is True
poll_for_run(instance, pipeline_run.run_id)
logs = instance.all_logs(pipeline_run.run_id)
assert (
len(
[
ev
for ev in logs
if ev.dagster_event.event_type_value == "STEP_MATERIALIZATION"
]
)
< 20
)
# soft termination
assert [ev for ev in logs if ev.dagster_event.event_type_value == "STEP_FAILURE"]
server_process.wait()
def get_repository_origin_from_kwargs(kwargs):
load_target = created_workspace_load_target(kwargs)
if isinstance(load_target, GrpcServerTarget):
return RepositoryGrpcServerOrigin(
host=load_target.host,
port=load_target.port,
socket=load_target.socket,
repository_name=kwargs["repository"],
)
return get_repository_python_origin_from_kwargs(kwargs)
def test_grpc_server_down():
with seven.TemporaryDirectory() as temp_dir:
with environ({'DAGSTER_HOME': temp_dir}):
down_grpc_repo_origin = RepositoryGrpcServerOrigin(
host='localhost',
port=find_free_port(),
socket=None,
repository_name='down_repo')
down_grpc_schedule_origin = down_grpc_repo_origin.get_schedule_origin(
'down_schedule')
instance = DagsterInstance.get()
result = sync_launch_scheduled_execution(down_grpc_schedule_origin)
assert isinstance(result, ScheduledExecutionFailed)
assert 'failed to connect to all addresses' in result.errors[
0].to_string()
ticks = instance.get_schedule_ticks(
down_grpc_schedule_origin.get_id())
assert ticks[0].status == ScheduleTickStatus.FAILURE
assert 'failed to connect to all addresses' in ticks[
0].error.message