def sync_launch_scheduled_execution(schedule_origin):
check.inst_param(schedule_origin, 'schedule_origin', SchedulePythonOrigin)
with get_temp_file_name() as output_file:
parts = ([
schedule_origin.executable_path,
'-m',
'dagster',
'api',
'launch_scheduled_execution',
output_file,
] + xplat_shlex_split(schedule_origin.get_repo_cli_args()) + [
'--schedule_name={}'.format(schedule_origin.schedule_name),
])
execute_command_in_subprocess(parts)
result = read_unary_response(output_file)
if isinstance(result, ScheduledExecutionResult):
return result
elif isinstance(result, IPCErrorMessage):
error = result.serializable_error_info
raise DagsterSubprocessError(
'Error in API subprocess: {message}\n\n{err}'.format(
message=result.message, err=error.to_string()),
subprocess_error_infos=[error],
)
else:
check.failed('Unexpected result {}'.format(result))
def sync_get_external_pipeline_subset(pipeline_handle, solid_selection=None):
check.inst_param(pipeline_handle, 'pipeline_handle', PipelineHandle)
check.opt_list_param(solid_selection, 'solid_selection', of_type=str)
location_handle = pipeline_handle.repository_handle.repository_location_handle
check.param_invariant(
isinstance(location_handle, PythonEnvRepositoryLocationHandle),
'pipeline_handle')
pointer = pipeline_handle.repository_handle.get_pointer()
with get_temp_file_name() as output_file:
parts = ([
location_handle.executable_path,
'-m',
'dagster',
'api',
'snapshot',
'pipeline_subset',
output_file,
] + xplat_shlex_split(pointer.get_cli_args()) +
[pipeline_handle.pipeline_name])
if solid_selection:
parts.append('--solid-selection={solid_selection}'.format(
solid_selection=json.dumps(solid_selection)))
execute_command_in_subprocess(parts)
external_pipeline_subset_result = read_unary_response(output_file)
check.inst(external_pipeline_subset_result,
ExternalPipelineSubsetResult)
return external_pipeline_subset_result
def sync_launch_scheduled_execution(schedule_origin, system_tz=None):
check.inst_param(schedule_origin, "schedule_origin", ExternalJobOrigin)
with get_temp_file_name() as output_file:
parts = (
[sys.executable, "-m", "dagster", "api", "launch_scheduled_execution", output_file,]
+ xplat_shlex_split(schedule_origin.get_repo_cli_args())
+ ["--schedule_name={}".format(schedule_origin.job_name)]
+ (["--override-system-timezone={}".format(system_tz)] if system_tz else [])
)
subprocess.check_call(parts)
result = read_unary_response(output_file)
if isinstance(result, ScheduledExecutionResult):
return result
elif isinstance(result, IPCErrorMessage):
error = result.serializable_error_info
raise DagsterSubprocessError(
"Error in API subprocess: {message}\n\n{err}".format(
message=result.message, err=error.to_string()
),
subprocess_error_infos=[error],
)
else:
check.failed("Unexpected result {}".format(result))
def sync_get_external_execution_plan(
pipeline_handle,
environment_dict,
mode,
snapshot_id,
solid_selection=None,
step_keys_to_execute=None,
):
check.inst_param(pipeline_handle, 'pipeline_handle', PipelineHandle)
check.opt_list_param(solid_selection, 'solid_selection', of_type=str)
check.dict_param(environment_dict, 'environment_dict')
check.str_param(mode, 'mode')
check.opt_list_param(step_keys_to_execute,
'step_keys_to_execute',
of_type=str)
check.str_param(snapshot_id, 'snapshot_id')
pointer = pipeline_handle.repository_handle.get_pointer()
location_handle = pipeline_handle.repository_handle.repository_location_handle
check.param_invariant(
isinstance(location_handle, PythonEnvRepositoryLocationHandle),
'pipeline_handle')
with get_temp_file_name() as output_file:
parts = ([
location_handle.executable_path,
'-m',
'dagster',
'api',
'snapshot',
'execution_plan',
output_file,
] + xplat_shlex_split(pointer.get_cli_args()) + [
pipeline_handle.pipeline_name,
'--environment-dict={environment_dict}'.format(
environment_dict=json.dumps(environment_dict)),
'--mode={mode}'.format(mode=mode),
'--snapshot-id={snapshot_id}'.format(snapshot_id=snapshot_id),
])
if solid_selection:
parts.append('--solid-selection={solid_selection}'.format(
solid_selection=json.dumps(solid_selection)))
if step_keys_to_execute:
parts.append(
'--step-keys-to-execute={step_keys_to_execute}'.format(
step_keys_to_execute=json.dumps(step_keys_to_execute)))
execute_command_in_subprocess(parts)
execution_plan_snapshot = read_unary_response(output_file)
check.inst(execution_plan_snapshot, ExecutionPlanSnapshot)
return execution_plan_snapshot
def wait_for_grpc_server(server_process, ipc_output_file, timeout=15):
event = read_unary_response(ipc_output_file, timeout=timeout, ipc_process=server_process)
if isinstance(event, GrpcServerFailedToBindEvent):
raise CouldNotBindGrpcServerToAddress()
elif isinstance(event, GrpcServerStartedEvent):
return True
else:
raise Exception(
"Received unexpected IPC event from gRPC Server: {event}".format(event=event)
)
def wait_for_grpc_server(server_process, ipc_output_file, timeout=60):
event = read_unary_response(ipc_output_file,
timeout=timeout,
ipc_process=server_process)
if isinstance(event, GrpcServerFailedToBindEvent):
raise CouldNotBindGrpcServerToAddress()
elif isinstance(event, GrpcServerLoadErrorEvent):
raise DagsterUserCodeProcessError(
event.error_info.to_string(),
user_code_process_error_infos=[event.error_info])
elif isinstance(event, GrpcServerStartedEvent):
return True
else:
raise Exception(
"Received unexpected IPC event from gRPC Server: {event}".format(
event=event))
def execute_unary_api_cli_command(executable_path, command_name, input_obj):
with get_temp_file_name() as input_file, get_temp_file_name(
) as output_file:
parts = [
executable_path,
'-m',
'dagster',
'api',
command_name,
input_file,
output_file,
]
write_unary_input(input_file, input_obj)
execute_command_in_subprocess(parts)
return read_unary_response(output_file)
def sync_list_repositories(executable_path, python_file, module_name):
from dagster.cli.api import ListRepositoriesResponse
with get_temp_file_name() as output_file:
parts = [
executable_path,
'-m',
'dagster',
'api',
'snapshot',
'list_repositories',
output_file,
] + (['-f', python_file]
if python_file else ['--module-name', module_name])
execute_command_in_subprocess(parts)
response = read_unary_response(output_file)
return check.inst(response, ListRepositoriesResponse)
def sync_get_external_repositories(repository_location_handle):
check.inst_param(
repository_location_handle, 'repository_location_handle', RepositoryLocationHandle,
)
check.param_invariant(
isinstance(repository_location_handle, PythonEnvRepositoryLocationHandle),
'repository_location_handle',
)
repos = []
for key, pointer in repository_location_handle.repository_code_pointer_dict.items():
with get_temp_file_name() as output_file:
parts = [
repository_location_handle.executable_path,
'-m',
'dagster',
'api',
'snapshot',
'repository',
output_file,
] + xplat_shlex_split(pointer.get_cli_args())
execute_command_in_subprocess(parts)
external_repository_data = read_unary_response(output_file)
check.inst(external_repository_data, ExternalRepositoryData)
repository_handle = RepositoryHandle(
repository_name=external_repository_data.name,
repository_key=key,
repository_location_handle=repository_location_handle,
)
repos.append(ExternalRepository(external_repository_data, repository_handle))
return repos
