def sync_get_external_repositories_grpc(repository_location_handle):
check.inst_param(
repository_location_handle, 'repository_location_handle', PythonEnvRepositoryLocationHandle,
)
repos = []
with ephemeral_grpc_api_client(
python_executable_path=repository_location_handle.executable_path
) as api_client:
for key, pointer in repository_location_handle.repository_code_pointer_dict.items():
external_repository_data = check.inst(
api_client.external_repository(
repository_python_origin=RepositoryPythonOrigin(
repository_location_handle.executable_path, pointer
)
),
ExternalRepositoryData,
)
repos.append(
ExternalRepository(
external_repository_data,
RepositoryHandle(
repository_name=external_repository_data.name,
repository_key=key,
repository_location_handle=repository_location_handle,
),
)
)
return repos
def cli_api_execute_run_grpc(instance_ref, pipeline_origin, pipeline_run):
check.inst_param(instance_ref, 'instance_ref', InstanceRef)
check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
instance = DagsterInstance.from_ref(instance_ref)
yield instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'.format(
pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id
),
pipeline_run,
engine_event_data=EngineEventData(marker_start='cli_api_subprocess_init'),
)
with ephemeral_grpc_api_client() as api_client:
execute_run_args = ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance_ref,
)
for event in api_client.execute_run(execute_run_args=execute_run_args):
if isinstance(event, IPCErrorMessage):
instance.report_engine_event(
event.message,
pipeline_run=pipeline_run,
engine_event_data=EngineEventData(
marker_end='cli_api_subprocess_init', error=event.serializable_error_info
),
)
instance.report_run_failed(pipeline_run)
return
yield event
def test_streaming_terminate():
with ephemeral_grpc_api_client() as api_client:
streaming_results = []
stream_events_result_thread = threading.Thread(
target=_stream_events_target, args=[streaming_results, api_client])
stream_events_result_thread.daemon = True
stream_events_result_thread.start()
start = time.time()
while not streaming_results:
if time.time() - start > 15:
raise Exception("Timed out waiting for streaming results")
time.sleep(0.001)
try:
api_client.shutdown_server()
except DagsterUserCodeUnreachableError:
# shutting down sometimes happens so fast that it terminates the calling RPC
pass
stream_events_result_thread.join(timeout=90)
assert not stream_events_result_thread.is_alive()
assert len(streaming_results) == 10000
api_client._server_process.wait() # pylint: disable=protected-access
assert api_client._server_process.poll() == 0 # pylint: disable=protected-access
def sync_list_repositories_ephemeral_grpc(
executable_path,
python_file,
module_name,
working_directory,
attribute,
package_name,
):
from dagster.grpc.client import ephemeral_grpc_api_client
check.str_param(executable_path, "executable_path")
check.opt_str_param(python_file, "python_file")
check.opt_str_param(module_name, "module_name")
check.opt_str_param(working_directory, "working_directory")
check.opt_str_param(package_name, "package_name")
with ephemeral_grpc_api_client(loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
module_name=module_name,
python_file=python_file,
working_directory=working_directory,
attribute=attribute,
package_name=package_name,
)) as api_client:
return sync_list_repositories_grpc(api_client)
def sync_get_external_pipeline_subset_ephemeral_grpc(pipeline_origin, solid_selection=None):
check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin)
check.opt_list_param(solid_selection, 'solid_selection', of_type=str)
with ephemeral_grpc_api_client(
loadable_target_origin=LoadableTargetOrigin(executable_path=pipeline_origin.executable_path)
) as api_client:
return sync_get_external_pipeline_subset_grpc(api_client, pipeline_origin, solid_selection)
def sync_get_external_job_params_ephemeral_grpc(instance, repository_handle,
name):
from dagster.grpc.client import ephemeral_grpc_api_client
origin = repository_handle.get_external_origin()
with ephemeral_grpc_api_client(origin.repository_location_origin.
loadable_target_origin) as api_client:
return sync_get_external_job_params_grpc(api_client, instance,
repository_handle, name)
def sync_get_external_schedule_execution_data_ephemeral_grpc(
instance, repository_handle, schedule_name
):
origin = repository_handle.get_origin()
with ephemeral_grpc_api_client(
LoadableTargetOrigin(executable_path=origin.executable_path)
) as api_client:
return sync_get_external_schedule_execution_data_grpc(
api_client, instance, repository_handle, schedule_name
)
def sync_get_external_job_params_ephemeral_grpc(instance, repository_handle,
name):
from dagster.grpc.client import ephemeral_grpc_api_client
origin = repository_handle.get_origin()
with ephemeral_grpc_api_client(
LoadableTargetOrigin(
executable_path=origin.executable_path)) as api_client:
return sync_get_external_job_params_grpc(api_client, instance,
repository_handle, name)
def sync_get_external_sensor_execution_data_ephemeral_grpc(
instance, repository_handle, sensor_name, last_completion_time):
from dagster.grpc.client import ephemeral_grpc_api_client
origin = repository_handle.get_external_origin()
with ephemeral_grpc_api_client(origin.repository_location_origin.
loadable_target_origin) as api_client:
return sync_get_external_sensor_execution_data_grpc(
api_client, instance, repository_handle, sensor_name,
last_completion_time)
def sync_get_external_partition_names_ephemeral_grpc(repository_handle, partition_set_name):
check.inst_param(repository_handle, 'repository_handle', RepositoryHandle)
check.str_param(partition_set_name, 'partition_set_name')
repository_origin = repository_handle.get_origin()
with ephemeral_grpc_api_client(
LoadableTargetOrigin(executable_path=repository_origin.executable_path)
) as api_client:
return sync_get_external_partition_names_grpc(
api_client, repository_handle, partition_set_name
)
def sync_get_external_pipeline_subset_grpc(pipeline_origin,
solid_selection=None):
check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin)
check.opt_list_param(solid_selection, 'solid_selection', of_type=str)
with ephemeral_grpc_api_client(python_executable_path=pipeline_origin.
executable_path) as api_client:
return check.inst(
api_client.external_pipeline_subset(
pipeline_subset_snapshot_args=PipelineSubsetSnapshotArgs(
pipeline_origin=pipeline_origin,
solid_selection=solid_selection), ),
ExternalPipelineSubsetResult,
)
def sync_get_external_partition_names_grpc(repository_handle, partition_set_name):
check.inst_param(repository_handle, 'repository_handle', RepositoryHandle)
check.str_param(partition_set_name, 'partition_set_name')
repository_origin = repository_handle.get_origin()
with ephemeral_grpc_api_client(
python_executable_path=repository_origin.executable_path
) as api_client:
return check.inst(
api_client.external_partition_names(
partition_names_args=PartitionNamesArgs(
repository_origin=repository_origin, partition_set_name=partition_set_name,
),
),
(ExternalPartitionNamesData, ExternalPartitionExecutionErrorData),
)
def sync_get_external_schedule_execution_data_ephemeral_grpc(
instance: DagsterInstance,
repository_handle: RepositoryHandle,
schedule_name: str,
scheduled_execution_time: Any,
):
from dagster.grpc.client import ephemeral_grpc_api_client
origin = repository_handle.get_external_origin()
with ephemeral_grpc_api_client(origin.repository_location_origin.
loadable_target_origin) as api_client:
return sync_get_external_schedule_execution_data_grpc(
api_client,
instance,
repository_handle,
schedule_name,
scheduled_execution_time,
)
def test_streaming_terminate():
with ephemeral_grpc_api_client() as api_client:
streaming_results = []
stream_events_result_thread = threading.Thread(
target=_stream_events_target, args=[streaming_results, api_client])
stream_events_result_thread.daemon = True
stream_events_result_thread.start()
while not streaming_results:
time.sleep(0.001)
res = api_client.shutdown_server()
assert res.success
assert res.serializable_error_info is None
stream_events_result_thread.join()
assert len(streaming_results) == 100000
api_client._server_process.wait() # pylint: disable=protected-access
assert api_client._server_process.poll() == 0 # pylint: disable=protected-access
def sync_list_repositories_ephemeral_grpc(executable_path, python_file,
module_name, working_directory):
from dagster.grpc.client import ephemeral_grpc_api_client
from dagster.grpc.types import LoadableTargetOrigin
check.str_param(executable_path, 'executable_path')
check.opt_str_param(python_file, 'python_file')
check.opt_str_param(module_name, 'module_name')
check.opt_str_param(working_directory, 'working_directory')
with ephemeral_grpc_api_client(loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
module_name=module_name,
python_file=python_file,
working_directory=working_directory,
attribute=None,
)) as api_client:
return sync_list_repositories_grpc(api_client)
def sync_get_external_schedule_execution_data_grpc(instance, repository_handle,
schedule_name):
check.inst_param(repository_handle, 'repository_handle', RepositoryHandle)
check.str_param(schedule_name, 'schedule_name')
origin = repository_handle.get_origin()
with ephemeral_grpc_api_client(
python_executable_path=origin.executable_path) as api_client:
return check.inst(
api_client.external_schedule_execution(
external_schedule_execution_args=ExternalScheduleExecutionArgs(
repository_origin=origin,
instance_ref=instance.get_ref(),
schedule_name=schedule_name,
)),
(ExternalScheduleExecutionData,
ExternalScheduleExecutionErrorData),
)
def sync_get_external_schedule_execution_data_ephemeral_grpc(
instance,
repository_handle,
schedule_name,
schedule_execution_data_mode,
scheduled_execution_datetime_utc,
):
from dagster.grpc.client import ephemeral_grpc_api_client
origin = repository_handle.get_origin()
with ephemeral_grpc_api_client(
LoadableTargetOrigin(executable_path=origin.executable_path)
) as api_client:
return sync_get_external_schedule_execution_data_grpc(
api_client,
instance,
repository_handle,
schedule_name,
schedule_execution_data_mode,
scheduled_execution_datetime_utc,
)
def sync_list_repositories_grpc(executable_path, python_file, module_name,
working_directory):
from dagster.grpc.client import ephemeral_grpc_api_client
from dagster.grpc.types import ListRepositoriesArgs, ListRepositoriesResponse
check.str_param(executable_path, 'executable_path')
check.opt_str_param(python_file, 'python_file')
check.opt_str_param(module_name, 'module_name')
check.opt_str_param(working_directory, 'working_directory')
with ephemeral_grpc_api_client(
python_executable_path=executable_path) as api_client:
return check.inst(
api_client.list_repositories(
ListRepositoriesArgs(
module_name=module_name,
python_file=python_file,
working_directory=working_directory,
)),
ListRepositoriesResponse,
)
