def sync_launch_scheduled_execution(schedule_origin):
check.inst_param(schedule_origin, 'schedule_origin', SchedulePythonOrigin)
with get_temp_file_name() as output_file:
parts = ([
schedule_origin.executable_path,
'-m',
'dagster',
'api',
'launch_scheduled_execution',
output_file,
] + xplat_shlex_split(schedule_origin.get_repo_cli_args()) + [
'--schedule_name={}'.format(schedule_origin.schedule_name),
])
execute_command_in_subprocess(parts)
result = read_unary_response(output_file)
if isinstance(result, ScheduledExecutionResult):
return result
elif isinstance(result, IPCErrorMessage):
error = result.serializable_error_info
raise DagsterSubprocessError(
'Error in API subprocess: {message}\n\n{err}'.format(
message=result.message, err=error.to_string()),
subprocess_error_infos=[error],
)
else:
check.failed('Unexpected result {}'.format(result))
def sync_get_external_pipeline_subset(pipeline_handle, solid_selection=None):
check.inst_param(pipeline_handle, 'pipeline_handle', PipelineHandle)
check.opt_list_param(solid_selection, 'solid_selection', of_type=str)
location_handle = pipeline_handle.repository_handle.repository_location_handle
check.param_invariant(
isinstance(location_handle, PythonEnvRepositoryLocationHandle),
'pipeline_handle')
pointer = pipeline_handle.repository_handle.get_pointer()
with get_temp_file_name() as output_file:
parts = ([
location_handle.executable_path,
'-m',
'dagster',
'api',
'snapshot',
'pipeline_subset',
output_file,
] + xplat_shlex_split(pointer.get_cli_args()) +
[pipeline_handle.pipeline_name])
if solid_selection:
parts.append('--solid-selection={solid_selection}'.format(
solid_selection=json.dumps(solid_selection)))
execute_command_in_subprocess(parts)
external_pipeline_subset_result = read_unary_response(output_file)
check.inst(external_pipeline_subset_result,
ExternalPipelineSubsetResult)
return external_pipeline_subset_result
def sync_launch_scheduled_execution(schedule_origin, system_tz=None):
check.inst_param(schedule_origin, "schedule_origin", ExternalJobOrigin)
with get_temp_file_name() as output_file:
parts = (
[sys.executable, "-m", "dagster", "api", "launch_scheduled_execution", output_file,]
+ xplat_shlex_split(schedule_origin.get_repo_cli_args())
+ ["--schedule_name={}".format(schedule_origin.job_name)]
+ (["--override-system-timezone={}".format(system_tz)] if system_tz else [])
)
subprocess.check_call(parts)
result = read_unary_response(output_file)
if isinstance(result, ScheduledExecutionResult):
return result
elif isinstance(result, IPCErrorMessage):
error = result.serializable_error_info
raise DagsterSubprocessError(
"Error in API subprocess: {message}\n\n{err}".format(
message=result.message, err=error.to_string()
),
subprocess_error_infos=[error],
)
else:
check.failed("Unexpected result {}".format(result))
def cli_api_execute_run(output_file, instance, repo_cli_args, pipeline_run):
check.str_param(output_file, 'output_file')
check.inst_param(instance, 'instance', DagsterInstance)
check.str_param(repo_cli_args, 'repo_cli_args')
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
parts = ([
'dagster',
'api',
'execute_run',
output_file,
] + xplat_shlex_split(repo_cli_args) + [
'--instance-ref',
'{instance_ref}'.format(
instance_ref=serialize_dagster_namedtuple(instance.get_ref())),
'--pipeline-run',
'{pipeline_run}'.format(
pipeline_run=serialize_dagster_namedtuple(pipeline_run)),
])
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='cli_api_subprocess_init'),
)
return open_ipc_subprocess(parts)
def dbt_test_solid(context):
executable_path = context.solid_config['dbt_executable']
if not distutils.spawn.find_executable(executable_path):
raise Failure(
'Could not find dbt executable at "{executable_path}". Please ensure that '
'dbt is installed and on the PATH.'.format(
executable_path=executable_path))
cmd = '{executable_path} test --project-dir {project_dir}'.format(
executable_path=executable_path, project_dir=project_dir)
if profiles_dir:
cmd += ' --profiles-dir {}'.format(profiles_dir)
args = xplat_shlex_split(cmd)
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
for line in io.TextIOWrapper(proc.stdout, encoding='utf-8'):
text = line.rstrip()
if not text:
continue
# print to stdout
print(text)
# remove colors
text = ANSI_ESCAPE.sub('', text)
expt = try_parse_test(text)
if expt:
yield expt
yield Output(value=None, output_name='test_complete')
def sync_get_external_execution_plan(
pipeline_handle,
environment_dict,
mode,
snapshot_id,
solid_selection=None,
step_keys_to_execute=None,
):
check.inst_param(pipeline_handle, 'pipeline_handle', PipelineHandle)
check.opt_list_param(solid_selection, 'solid_selection', of_type=str)
check.dict_param(environment_dict, 'environment_dict')
check.str_param(mode, 'mode')
check.opt_list_param(step_keys_to_execute,
'step_keys_to_execute',
of_type=str)
check.str_param(snapshot_id, 'snapshot_id')
pointer = pipeline_handle.repository_handle.get_pointer()
location_handle = pipeline_handle.repository_handle.repository_location_handle
check.param_invariant(
isinstance(location_handle, PythonEnvRepositoryLocationHandle),
'pipeline_handle')
with get_temp_file_name() as output_file:
parts = ([
location_handle.executable_path,
'-m',
'dagster',
'api',
'snapshot',
'execution_plan',
output_file,
] + xplat_shlex_split(pointer.get_cli_args()) + [
pipeline_handle.pipeline_name,
'--environment-dict={environment_dict}'.format(
environment_dict=json.dumps(environment_dict)),
'--mode={mode}'.format(mode=mode),
'--snapshot-id={snapshot_id}'.format(snapshot_id=snapshot_id),
])
if solid_selection:
parts.append('--solid-selection={solid_selection}'.format(
solid_selection=json.dumps(solid_selection)))
if step_keys_to_execute:
parts.append(
'--step-keys-to-execute={step_keys_to_execute}'.format(
step_keys_to_execute=json.dumps(step_keys_to_execute)))
execute_command_in_subprocess(parts)
execution_plan_snapshot = read_unary_response(output_file)
check.inst(execution_plan_snapshot, ExecutionPlanSnapshot)
return execution_plan_snapshot
def cli_api_execute_run(output_file, instance, repository_handle,
pipeline_run):
check.str_param(output_file, 'output_file')
check.inst_param(instance, 'instance', DagsterInstance)
check.inst_param(repository_handle, 'repository_handle', RepositoryHandle)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
pointer = repository_handle.get_pointer()
location_handle = repository_handle.repository_location_handle
if isinstance(location_handle, PythonEnvRepositoryLocationHandle):
executable_path = location_handle.executable_path
elif isinstance(location_handle, InProcessRepositoryLocationHandle):
# [legacy] default to using sys.executable for the in process
# location handle
executable_path = sys.executable
else:
raise DagsterInvariantViolationError(
"Unable to resolve executable_path for repository location handle of type {}"
.format(location_handle.__class__))
executable_path = (location_handle.executable_path if isinstance(
location_handle, PythonEnvRepositoryLocationHandle) else
sys.executable)
parts = (
[executable_path, '-m', 'dagster', 'api', 'execute_run', output_file] +
xplat_shlex_split(pointer.get_cli_args()) + [
'--instance-ref',
'{instance_ref}'.format(
instance_ref=serialize_dagster_namedtuple(instance.get_ref())),
'--pipeline-run-id',
'{pipeline_run_id}'.format(pipeline_run_id=pipeline_run.run_id),
])
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='cli_api_subprocess_init'),
)
return open_ipc_subprocess(parts)
def dbt_solid(context):
executable_path = context.solid_config['dbt_executable']
if not distutils.spawn.find_executable(executable_path):
raise Failure(
'Could not find dbt executable at "{executable_path}". Please ensure that '
'dbt is installed and on the PATH.'.format(
executable_path=executable_path))
cmd = '{executable_path} run --project-dir {project_dir}'.format(
executable_path=executable_path, project_dir=project_dir)
if profiles_dir:
cmd += ' --profiles-dir {profiles_dir}'.format(
profiles_dir=profiles_dir)
args = xplat_shlex_split(cmd)
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
# if https://github.com/fishtown-analytics/dbt/issues/1237 gets done
# we should definitely switch to parsing the json output, as that
# would be much more reliable/resilient
for line in io.TextIOWrapper(proc.stdout, encoding='utf-8'):
text = line.rstrip()
if not text:
continue
# print to stdout
print(text)
# remove colors
text = ANSI_ESCAPE.sub('', text)
mat = try_parse_run(text)
if mat:
yield mat
proc.wait()
if proc.returncode != 0:
raise Failure('Dbt invocation errored.')
yield Output(value=None, output_name='run_complete')
def sync_get_external_repositories(repository_location_handle):
check.inst_param(
repository_location_handle, 'repository_location_handle', RepositoryLocationHandle,
)
check.param_invariant(
isinstance(repository_location_handle, PythonEnvRepositoryLocationHandle),
'repository_location_handle',
)
repos = []
for key, pointer in repository_location_handle.repository_code_pointer_dict.items():
with get_temp_file_name() as output_file:
parts = [
repository_location_handle.executable_path,
'-m',
'dagster',
'api',
'snapshot',
'repository',
output_file,
] + xplat_shlex_split(pointer.get_cli_args())
execute_command_in_subprocess(parts)
external_repository_data = read_unary_response(output_file)
check.inst(external_repository_data, ExternalRepositoryData)
repository_handle = RepositoryHandle(
repository_name=external_repository_data.name,
repository_key=key,
repository_location_handle=repository_location_handle,
)
repos.append(ExternalRepository(external_repository_data, repository_handle))
return repos
def sync_get_external_repository(location_handle):
check.inst_param(location_handle, 'location_handle', LocationHandle)
with get_temp_file_name() as output_file:
parts = ['dagster', 'api', 'snapshot', 'repository', output_file] + xplat_shlex_split(
location_handle.pointer.get_cli_args()
)
returncode = subprocess.check_call(parts)
check.invariant(returncode == 0, 'dagster api cli invocation did not complete successfully')
messages = list(ipc_read_event_stream(output_file))
check.invariant(len(messages) == 1)
external_repository_data = messages[0]
check.inst(external_repository_data, ExternalRepositoryData)
return ExternalRepository(
external_repository_data,
RepositoryHandle(external_repository_data.name, location_handle),
)