def test_multi():
instance = DagsterInstance.local_temp()
pipeline_name = 'foo_pipeline'
pipeline_run = create_run_for_test(instance, pipeline_name=pipeline_name)
context = get_multiprocessing_context()
step_keys = ['A', 'B', 'C']
with instance.compute_log_manager.watch(pipeline_run):
print('outer 1')
print('outer 2')
print('outer 3')
for step_key in step_keys:
process = context.Process(target=execute_inner,
args=(step_key, pipeline_run,
instance.get_ref()))
process.start()
process.join()
for step_key in step_keys:
stdout = instance.compute_log_manager.read_logs_file(
pipeline_run.run_id, step_key, ComputeIOType.STDOUT)
assert normalize_file_content(
stdout.data) == expected_inner_output(step_key)
full_out = instance.compute_log_manager.read_logs_file(
pipeline_run.run_id, pipeline_name, ComputeIOType.STDOUT)
# The way that the multiprocess compute-logging interacts with pytest (which stubs out the
# sys.stdout fileno) makes this difficult to test. The pytest-captured stdout only captures
# the stdout from the outer process, not also the inner process
assert normalize_file_content(full_out.data).startswith(
expected_outer_prefix())
def __init__(self, instance):
self._multiprocessing_context = get_multiprocessing_context()
self._instance = instance
self._living_process_by_run_id = {}
self._processes_lock = self._multiprocessing_context.Lock()
gevent.spawn(self._check_for_zombies)
def __init__(self, instance, max_concurrent_runs):
check.inst_param(instance, 'instance', DagsterInstance)
self._delegate = SubprocessExecutionManager(instance)
self._max_concurrent_runs = check.int_param(max_concurrent_runs, 'max_concurrent_runs')
self._multiprocessing_context = get_multiprocessing_context()
self._queue = self._multiprocessing_context.JoinableQueue(maxsize=0)
gevent.spawn(self._clock)
def execute_child_process_command(command, return_process_events=False):
'''
This function polls the process until it returns a valid
item or returns PROCESS_DEAD_AND_QUEUE_EMPTY if it is in
a state where the process has terminated and the queue is empty.
If wait_mode is set to YIELD, it will yield None while the process is busy executing.
Warning: if the child process is in an infinite loop. This will
also infinitely loop.
'''
check.inst_param(command, 'command', ChildProcessCommand)
check.bool_param(return_process_events, 'return_process_events')
multiprocessing_context = get_multiprocessing_context()
queue = multiprocessing_context.Queue()
process = multiprocessing_context.Process(
target=execute_command_in_child_process, args=(queue, command)
)
process.start()
completed_properly = False
while not completed_properly:
event = get_next_event(process, queue)
# child process is busy executing, yield so we (the parent) can continue
# other work such as checking other child_process_commands
if event is None:
yield None
if event == PROCESS_DEAD_AND_QUEUE_EMPTY:
break
# If we are configured to return process events by the caller,
# yield that event to the caller
if return_process_events and isinstance(event, ChildProcessEvents):
yield event
if isinstance(event, ChildProcessDoneEvent):
completed_properly = True
elif isinstance(event, ChildProcessSystemErrorEvent):
raise ChildProcessException(
'Uncaught exception in process {pid} with message "{message}" and error info {error_info}'.format(
pid=event.pid, message=event.error_info.message, error_info=event.error_info
),
error_info=event.error_info,
)
elif not isinstance(event, ChildProcessEvents):
yield event
if not completed_properly:
# TODO Gather up stderr and the process exit code
raise ChildProcessCrashException()
process.join()
def bounded_parallel_executor(pipeline_context, step_contexts, limit):
pending_execution = list(step_contexts)
active_iters = {}
errors = {}
term_events = {}
stopping = False
while (not stopping and pending_execution) or active_iters:
try:
while len(active_iters
) < limit and pending_execution and not stopping:
step_context = pending_execution.pop(0)
step = step_context.step
term_events[step.key] = get_multiprocessing_context().Event()
active_iters[step.key] = execute_step_out_of_process(
step_context, step, errors, term_events)
empty_iters = []
for key, step_iter in active_iters.items():
try:
event_or_none = next(step_iter)
if event_or_none is None:
continue
else:
yield event_or_none
except StopIteration:
empty_iters.append(key)
for key in empty_iters:
del active_iters[key]
if term_events[key].is_set():
stopping = True
del term_events[key]
# In the very small chance that we get interrupted in this coordination section and not
# polling the subprocesses for events - try to clean up greacefully
except KeyboardInterrupt:
yield DagsterEvent.engine_event(
pipeline_context,
'Multiprocess engine: received KeyboardInterrupt - forwarding to active child processes',
EngineEventData.interrupted(list(term_events.keys())),
)
for event in term_events.values():
event.set()
errs = {pid: err for pid, err in errors.items() if err}
if errs:
raise DagsterSubprocessError(
'During multiprocess execution errors occured in child processes:\n{error_list}'
.format(error_list='\n'.join([
'In process {pid}: {err}'.format(pid=pid, err=err.to_string())
for pid, err in errs.items()
])),
subprocess_error_infos=list(errs.values()),
)
def execute_child_process_command(command):
'''Execute a ChildProcessCommand in a new process.
This function starts a new process whose execution target is a ChildProcessCommand wrapped by
_execute_command_in_child_process; polls the queue for events yielded by the child process
until the process dies and the queue is empty.
This function yields a complex set of objects to enable having multiple child process
executions in flight:
* None - nothing has happened, yielded to enable cooperative multitasking other iterators
* ChildProcessEvent - Family of objects that communicates state changes in the child process
* KeyboardInterrupt - Yielded in the case that an interrupt was recieved while
polling the child process. Yielded instead of raised to allow forwarding of the
interrupt to the child and completion of the iterator for this child and
any others that may be executing
* The actual values yielded by the child process command
Args:
command (ChildProcessCommand): The command to execute in the child process.
Warning: if the child process is in an infinite loop, this will
also infinitely loop.
'''
check.inst_param(command, 'command', ChildProcessCommand)
multiprocessing_context = get_multiprocessing_context()
queue = multiprocessing_context.Queue()
process = multiprocessing_context.Process(
target=_execute_command_in_child_process, args=(queue, command))
process.start()
completed_properly = False
while not completed_properly:
event = _poll_for_event(process, queue)
if event == PROCESS_DEAD_AND_QUEUE_EMPTY:
break
yield event
if isinstance(event,
(ChildProcessDoneEvent, ChildProcessSystemErrorEvent)):
completed_properly = True
if not completed_properly:
# TODO Gather up stderr and the process exit code
raise ChildProcessCrashException()
process.join()
def __init__(self):
self._multiprocessing_context = get_multiprocessing_context()
self._processes_lock = self._multiprocessing_context.Lock()
self._processes = []
# This is actually a reverse semaphore. We keep track of number of
# processes we have by releasing semaphore every time we start
# processing, we acquire after processing is finished
self._processing_semaphore = gevent.lock.Semaphore(0)
gevent.spawn(self._start_polling)
atexit.register(self._cleanup)
def execute_child_process_command(command, return_process_events=False):
check.inst_param(command, 'command', ChildProcessCommand)
check.bool_param(return_process_events, 'return_process_events')
multiprocessing_context = get_multiprocessing_context()
queue = multiprocessing_context.Queue()
process = multiprocessing_context.Process(
target=execute_command_in_child_process, args=(queue, command)
)
process.start()
completed_properly = False
while not completed_properly:
event = get_next_event(process, queue)
if event == PROCESS_DEAD_AND_QUEUE_EMPTY:
break
# If we are configured to return process events by the caller,
# yield that event to the caller
if return_process_events and isinstance(event, ChildProcessEvents):
yield event
if isinstance(event, ChildProcessDoneEvent):
completed_properly = True
elif isinstance(event, ChildProcessSystemErrorEvent):
raise ChildProcessException(
'Uncaught exception in process {pid} with message "{message}" and error info {error_info}'.format(
pid=event.pid, message=event.error_info.message, error_info=event.error_info
),
error_info=event.error_info,
)
elif not isinstance(event, ChildProcessEvents):
yield event
if not completed_properly:
# TODO Gather up stderr and the process exit code
raise ChildProcessCrashException()
process.join()
def execute(self, pipeline_context, execution_plan):
check.inst_param(pipeline_context, 'pipeline_context',
SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
limit = self.max_concurrent
yield DagsterEvent.engine_event(
pipeline_context,
'Executing steps using multiprocess engine: parent process (pid: {pid})'
.format(pid=os.getpid()),
event_specific_data=EngineEventData.multiprocess(
os.getpid(),
step_keys_to_execute=execution_plan.step_keys_to_execute),
)
# It would be good to implement a reference tracking algorithm here so we could
# garbage collection results that are no longer needed by any steps
# https://github.com/dagster-io/dagster/issues/811
with time_execution_scope() as timer_result:
active_execution = execution_plan.start(retries=self.retries)
active_iters = {}
errors = {}
term_events = {}
stopping = False
while (not stopping
and not active_execution.is_complete) or active_iters:
try:
# start iterators
while len(active_iters) < limit and not stopping:
steps = active_execution.get_steps_to_execute(
limit=(limit - len(active_iters)))
if not steps:
break
for step in steps:
step_context = pipeline_context.for_step(step)
term_events[
step.key] = get_multiprocessing_context(
).Event()
active_iters[
step.key] = self.execute_step_out_of_process(
step_context, step, errors, term_events)
# process active iterators
empty_iters = []
for key, step_iter in active_iters.items():
try:
event_or_none = next(step_iter)
if event_or_none is None:
continue
else:
yield event_or_none
active_execution.handle_event(event_or_none)
except StopIteration:
empty_iters.append(key)
# clear and mark complete finished iterators
for key in empty_iters:
del active_iters[key]
if term_events[key].is_set():
stopping = True
del term_events[key]
active_execution.verify_complete(pipeline_context, key)
# process skips from failures or uncovered inputs
for event in active_execution.skipped_step_events_iterator(
pipeline_context):
yield event
# In the very small chance that we get interrupted in this coordination section and not
# polling the subprocesses for events - try to clean up gracefully
except KeyboardInterrupt:
yield DagsterEvent.engine_event(
pipeline_context,
'Multiprocess engine: received KeyboardInterrupt - forwarding to active child processes',
EngineEventData.interrupted(list(term_events.keys())),
)
stopping = True
for event in term_events.values():
event.set()
errs = {pid: err for pid, err in errors.items() if err}
if errs:
raise DagsterSubprocessError(
'During multiprocess execution errors occurred in child processes:\n{error_list}'
.format(error_list='\n'.join([
'In process {pid}: {err}'.format(pid=pid,
err=err.to_string())
for pid, err in errs.items()
])),
subprocess_error_infos=list(errs.values()),
)
yield DagsterEvent.engine_event(
pipeline_context,
'Multiprocess engine: parent process exiting after {duration} (pid: {pid})'
.format(duration=format_duration(timer_result.millis),
pid=os.getpid()),
event_specific_data=EngineEventData.multiprocess(os.getpid()),
)
def execute_child_process_command(command, return_process_events=False):
'''Execute a ChildProcessCommand in a new process.
This function starts a new process whose execution target is a ChildProcessCommand wrapped by
_execute_command_in_child_process; polls the queue for events yielded by the child process
until the process dies and the queue is empty.
Args:
command (ChildProcessCommand): The command to execute in the child process.
return_process_events(Optional[bool]): Set this flag to yield the control events
(ChildProcessEvents) back to the caller of this function, in addition to any
non-control events. (default: False)
Warning: if the child process is in an infinite loop, this will
also infinitely loop.
'''
check.inst_param(command, 'command', ChildProcessCommand)
check.bool_param(return_process_events, 'return_process_events')
multiprocessing_context = get_multiprocessing_context()
queue = multiprocessing_context.Queue()
process = multiprocessing_context.Process(
target=_execute_command_in_child_process, args=(queue, command))
process.start()
completed_properly = False
while not completed_properly:
event = _poll_for_event(process, queue)
# child process is busy executing, yield so we (the parent) can continue
# other work such as checking other child_process_commands
if event is None:
yield None
if event == PROCESS_DEAD_AND_QUEUE_EMPTY:
break
# If we are configured to return process events by the caller,
# yield that event to the caller
if return_process_events and isinstance(event, ChildProcessEvents):
yield event
if isinstance(event, ChildProcessDoneEvent):
completed_properly = True
elif isinstance(event, ChildProcessSystemErrorEvent):
raise ChildProcessException(
'Uncaught exception in process {pid} with message "{message}" and error info {error_info}'
.format(pid=event.pid,
message=event.error_info.message,
error_info=event.error_info),
error_info=event.error_info,
)
elif not isinstance(event, ChildProcessEvents):
yield event
if not completed_properly:
# TODO Gather up stderr and the process exit code
raise ChildProcessCrashException()
process.join()
