def sync_launch_scheduled_execution(schedule_origin): check.inst_param(schedule_origin, 'schedule_origin', SchedulePythonOrigin) with get_temp_file_name() as output_file: parts = ([ schedule_origin.executable_path, '-m', 'dagster', 'api', 'launch_scheduled_execution', output_file, ] + xplat_shlex_split(schedule_origin.get_repo_cli_args()) + [ '--schedule_name={}'.format(schedule_origin.schedule_name), ]) execute_command_in_subprocess(parts) result = read_unary_response(output_file) if isinstance(result, ScheduledExecutionResult): return result elif isinstance(result, IPCErrorMessage): error = result.serializable_error_info raise DagsterSubprocessError( 'Error in API subprocess: {message}\n\n{err}'.format( message=result.message, err=error.to_string()), subprocess_error_infos=[error], ) else: check.failed('Unexpected result {}'.format(result))
def sync_get_external_pipeline_subset(pipeline_handle, solid_selection=None): check.inst_param(pipeline_handle, 'pipeline_handle', PipelineHandle) check.opt_list_param(solid_selection, 'solid_selection', of_type=str) location_handle = pipeline_handle.repository_handle.repository_location_handle check.param_invariant( isinstance(location_handle, PythonEnvRepositoryLocationHandle), 'pipeline_handle') pointer = pipeline_handle.repository_handle.get_pointer() with get_temp_file_name() as output_file: parts = ([ location_handle.executable_path, '-m', 'dagster', 'api', 'snapshot', 'pipeline_subset', output_file, ] + xplat_shlex_split(pointer.get_cli_args()) + [pipeline_handle.pipeline_name]) if solid_selection: parts.append('--solid-selection={solid_selection}'.format( solid_selection=json.dumps(solid_selection))) execute_command_in_subprocess(parts) external_pipeline_subset_result = read_unary_response(output_file) check.inst(external_pipeline_subset_result, ExternalPipelineSubsetResult) return external_pipeline_subset_result
def sync_launch_scheduled_execution(schedule_origin, system_tz=None): check.inst_param(schedule_origin, "schedule_origin", ExternalJobOrigin) with get_temp_file_name() as output_file: parts = ( [sys.executable, "-m", "dagster", "api", "launch_scheduled_execution", output_file,] + xplat_shlex_split(schedule_origin.get_repo_cli_args()) + ["--schedule_name={}".format(schedule_origin.job_name)] + (["--override-system-timezone={}".format(system_tz)] if system_tz else []) ) subprocess.check_call(parts) result = read_unary_response(output_file) if isinstance(result, ScheduledExecutionResult): return result elif isinstance(result, IPCErrorMessage): error = result.serializable_error_info raise DagsterSubprocessError( "Error in API subprocess: {message}\n\n{err}".format( message=result.message, err=error.to_string() ), subprocess_error_infos=[error], ) else: check.failed("Unexpected result {}".format(result))
def cli_api_execute_run(output_file, instance, repo_cli_args, pipeline_run): check.str_param(output_file, 'output_file') check.inst_param(instance, 'instance', DagsterInstance) check.str_param(repo_cli_args, 'repo_cli_args') check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) parts = ([ 'dagster', 'api', 'execute_run', output_file, ] + xplat_shlex_split(repo_cli_args) + [ '--instance-ref', '{instance_ref}'.format( instance_ref=serialize_dagster_namedtuple(instance.get_ref())), '--pipeline-run', '{pipeline_run}'.format( pipeline_run=serialize_dagster_namedtuple(pipeline_run)), ]) instance.report_engine_event( 'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).' .format(pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id), pipeline_run, engine_event_data=EngineEventData( marker_start='cli_api_subprocess_init'), ) return open_ipc_subprocess(parts)
def dbt_test_solid(context): executable_path = context.solid_config['dbt_executable'] if not distutils.spawn.find_executable(executable_path): raise Failure( 'Could not find dbt executable at "{executable_path}". Please ensure that ' 'dbt is installed and on the PATH.'.format( executable_path=executable_path)) cmd = '{executable_path} test --project-dir {project_dir}'.format( executable_path=executable_path, project_dir=project_dir) if profiles_dir: cmd += ' --profiles-dir {}'.format(profiles_dir) args = xplat_shlex_split(cmd) proc = subprocess.Popen(args, stdout=subprocess.PIPE) for line in io.TextIOWrapper(proc.stdout, encoding='utf-8'): text = line.rstrip() if not text: continue # print to stdout print(text) # remove colors text = ANSI_ESCAPE.sub('', text) expt = try_parse_test(text) if expt: yield expt yield Output(value=None, output_name='test_complete')
def sync_get_external_execution_plan( pipeline_handle, environment_dict, mode, snapshot_id, solid_selection=None, step_keys_to_execute=None, ): check.inst_param(pipeline_handle, 'pipeline_handle', PipelineHandle) check.opt_list_param(solid_selection, 'solid_selection', of_type=str) check.dict_param(environment_dict, 'environment_dict') check.str_param(mode, 'mode') check.opt_list_param(step_keys_to_execute, 'step_keys_to_execute', of_type=str) check.str_param(snapshot_id, 'snapshot_id') pointer = pipeline_handle.repository_handle.get_pointer() location_handle = pipeline_handle.repository_handle.repository_location_handle check.param_invariant( isinstance(location_handle, PythonEnvRepositoryLocationHandle), 'pipeline_handle') with get_temp_file_name() as output_file: parts = ([ location_handle.executable_path, '-m', 'dagster', 'api', 'snapshot', 'execution_plan', output_file, ] + xplat_shlex_split(pointer.get_cli_args()) + [ pipeline_handle.pipeline_name, '--environment-dict={environment_dict}'.format( environment_dict=json.dumps(environment_dict)), '--mode={mode}'.format(mode=mode), '--snapshot-id={snapshot_id}'.format(snapshot_id=snapshot_id), ]) if solid_selection: parts.append('--solid-selection={solid_selection}'.format( solid_selection=json.dumps(solid_selection))) if step_keys_to_execute: parts.append( '--step-keys-to-execute={step_keys_to_execute}'.format( step_keys_to_execute=json.dumps(step_keys_to_execute))) execute_command_in_subprocess(parts) execution_plan_snapshot = read_unary_response(output_file) check.inst(execution_plan_snapshot, ExecutionPlanSnapshot) return execution_plan_snapshot
def cli_api_execute_run(output_file, instance, repository_handle, pipeline_run): check.str_param(output_file, 'output_file') check.inst_param(instance, 'instance', DagsterInstance) check.inst_param(repository_handle, 'repository_handle', RepositoryHandle) check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) pointer = repository_handle.get_pointer() location_handle = repository_handle.repository_location_handle if isinstance(location_handle, PythonEnvRepositoryLocationHandle): executable_path = location_handle.executable_path elif isinstance(location_handle, InProcessRepositoryLocationHandle): # [legacy] default to using sys.executable for the in process # location handle executable_path = sys.executable else: raise DagsterInvariantViolationError( "Unable to resolve executable_path for repository location handle of type {}" .format(location_handle.__class__)) executable_path = (location_handle.executable_path if isinstance( location_handle, PythonEnvRepositoryLocationHandle) else sys.executable) parts = ( [executable_path, '-m', 'dagster', 'api', 'execute_run', output_file] + xplat_shlex_split(pointer.get_cli_args()) + [ '--instance-ref', '{instance_ref}'.format( instance_ref=serialize_dagster_namedtuple(instance.get_ref())), '--pipeline-run-id', '{pipeline_run_id}'.format(pipeline_run_id=pipeline_run.run_id), ]) instance.report_engine_event( 'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).' .format(pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id), pipeline_run, engine_event_data=EngineEventData( marker_start='cli_api_subprocess_init'), ) return open_ipc_subprocess(parts)
def dbt_solid(context): executable_path = context.solid_config['dbt_executable'] if not distutils.spawn.find_executable(executable_path): raise Failure( 'Could not find dbt executable at "{executable_path}". Please ensure that ' 'dbt is installed and on the PATH.'.format( executable_path=executable_path)) cmd = '{executable_path} run --project-dir {project_dir}'.format( executable_path=executable_path, project_dir=project_dir) if profiles_dir: cmd += ' --profiles-dir {profiles_dir}'.format( profiles_dir=profiles_dir) args = xplat_shlex_split(cmd) proc = subprocess.Popen(args, stdout=subprocess.PIPE) # if https://github.com/fishtown-analytics/dbt/issues/1237 gets done # we should definitely switch to parsing the json output, as that # would be much more reliable/resilient for line in io.TextIOWrapper(proc.stdout, encoding='utf-8'): text = line.rstrip() if not text: continue # print to stdout print(text) # remove colors text = ANSI_ESCAPE.sub('', text) mat = try_parse_run(text) if mat: yield mat proc.wait() if proc.returncode != 0: raise Failure('Dbt invocation errored.') yield Output(value=None, output_name='run_complete')
def sync_get_external_repositories(repository_location_handle): check.inst_param( repository_location_handle, 'repository_location_handle', RepositoryLocationHandle, ) check.param_invariant( isinstance(repository_location_handle, PythonEnvRepositoryLocationHandle), 'repository_location_handle', ) repos = [] for key, pointer in repository_location_handle.repository_code_pointer_dict.items(): with get_temp_file_name() as output_file: parts = [ repository_location_handle.executable_path, '-m', 'dagster', 'api', 'snapshot', 'repository', output_file, ] + xplat_shlex_split(pointer.get_cli_args()) execute_command_in_subprocess(parts) external_repository_data = read_unary_response(output_file) check.inst(external_repository_data, ExternalRepositoryData) repository_handle = RepositoryHandle( repository_name=external_repository_data.name, repository_key=key, repository_location_handle=repository_location_handle, ) repos.append(ExternalRepository(external_repository_data, repository_handle)) return repos
def sync_get_external_repository(location_handle): check.inst_param(location_handle, 'location_handle', LocationHandle) with get_temp_file_name() as output_file: parts = ['dagster', 'api', 'snapshot', 'repository', output_file] + xplat_shlex_split( location_handle.pointer.get_cli_args() ) returncode = subprocess.check_call(parts) check.invariant(returncode == 0, 'dagster api cli invocation did not complete successfully') messages = list(ipc_read_event_stream(output_file)) check.invariant(len(messages) == 1) external_repository_data = messages[0] check.inst(external_repository_data, ExternalRepositoryData) return ExternalRepository( external_repository_data, RepositoryHandle(external_repository_data.name, location_handle), )