def execute_run_command(input_json):
with capture_interrupts():
args = deserialize_as(input_json, ExecuteRunArgs)
recon_pipeline = recon_pipeline_from_origin(args.pipeline_origin)
with (DagsterInstance.from_ref(args.instance_ref)
if args.instance_ref else DagsterInstance.get()) as instance:
buffer = []
def send_to_buffer(event):
buffer.append(serialize_dagster_namedtuple(event))
return_code = _execute_run_command_body(
recon_pipeline,
args.pipeline_run_id,
instance,
send_to_buffer,
set_exit_code_on_failure=args.set_exit_code_on_failure
or False,
)
for line in buffer:
click.echo(line)
if return_code != 0:
sys.exit(return_code)
def _execute_command_in_child_process(event_queue, command):
"""Wraps the execution of a ChildProcessCommand.
Handles errors and communicates across a queue with the parent process."""
check.inst_param(command, "command", ChildProcessCommand)
with capture_interrupts():
pid = os.getpid()
event_queue.put(ChildProcessStartEvent(pid=pid))
try:
for step_event in command.execute():
event_queue.put(step_event)
event_queue.put(ChildProcessDoneEvent(pid=pid))
except (
Exception,
KeyboardInterrupt,
DagsterExecutionInterruptedError,
):
event_queue.put(
ChildProcessSystemErrorEvent(
pid=pid,
error_info=serializable_error_info_from_exc_info(
sys.exc_info())))
def run_command():
with capture_interrupts():
with DagsterInstance.get() as instance:
if instance.is_ephemeral:
raise Exception(
"dagster-daemon can't run using an in-memory instance. Make sure "
"the DAGSTER_HOME environment variable has been set correctly and that "
"you have created a dagster.yaml file there."
)
with daemon_controller_from_instance(instance) as controller:
start_time = pendulum.now("UTC")
while True:
# Wait until a daemon has been unhealthy for a long period of time
# before potentially restarting it due to a hanging or failed daemon
with raise_interrupts_as(KeyboardInterrupt):
time.sleep(1)
if (
pendulum.now("UTC") - start_time
).total_seconds() < 2 * DAEMON_HEARTBEAT_TOLERANCE_SECONDS:
continue
controller.check_daemons()
start_time = pendulum.now("UTC")
def __iter__(self):
# Since interrupts can't be raised at arbitrary points safely, delay them until designated
# checkpoints during the execution.
# To be maximally certain that interrupts are always caught during an execution process,
# you can safely add an additional `with capture_interrupts()` at the very beginning of the
# process that performs the execution.
with capture_interrupts():
yield from self.execution_context_manager.prepare_context()
self.pipeline_context = self.execution_context_manager.get_context(
)
generator_closed = False
try:
if self.pipeline_context: # False if we had a pipeline init failure
yield from self.iterator(
execution_plan=self.execution_plan,
pipeline_context=self.pipeline_context,
)
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
raise
finally:
for event in self.execution_context_manager.shutdown_context():
if not generator_closed:
yield event
def execute_step_command(input_json):
with capture_interrupts():
args = check.inst(deserialize_json_to_dagster_namedtuple(input_json),
ExecuteStepArgs)
with (DagsterInstance.from_ref(args.instance_ref)
if args.instance_ref else DagsterInstance.get()) as instance:
pipeline_run = instance.get_run_by_id(args.pipeline_run_id)
check.inst(
pipeline_run,
PipelineRun,
"Pipeline run with id '{}' not found for step execution".
format(args.pipeline_run_id),
)
if args.should_verify_step:
success = verify_step(
instance,
pipeline_run,
args.known_state.get_retry_state(),
args.step_keys_to_execute,
)
if not success:
return
recon_pipeline = recon_pipeline_from_origin(args.pipeline_origin)
execution_plan = create_execution_plan(
recon_pipeline.subset_for_execution_from_existing_pipeline(
pipeline_run.solids_to_execute),
run_config=pipeline_run.run_config,
step_keys_to_execute=args.step_keys_to_execute,
mode=pipeline_run.mode,
known_state=args.known_state,
)
buff = []
# Flag that the step execution is skipped
if should_skip_step(execution_plan,
instance=instance,
run_id=pipeline_run.run_id):
click.echo(serialize_dagster_namedtuple(
StepExecutionSkipped()))
return
for event in execute_plan_iterator(
execution_plan,
pipeline_run,
instance,
run_config=pipeline_run.run_config,
retry_mode=args.retry_mode,
):
buff.append(serialize_dagster_namedtuple(event))
for line in buff:
click.echo(line)
def start_run_in_subprocess(serialized_execute_run_args, recon_pipeline,
event_queue, termination_event):
with capture_interrupts():
_run_in_subprocess(
serialized_execute_run_args,
recon_pipeline,
termination_event,
subprocess_status_handler=event_queue.put,
run_event_handler=lambda x: None,
)
def pipeline_execute_command(**kwargs):
with capture_interrupts():
if is_dagster_home_set():
with DagsterInstance.get() as instance:
execute_execute_command(instance, kwargs)
else:
warnings.warn(
"DAGSTER_HOME is not set, no metadata will be recorded for this execution.\n",
)
execute_execute_command(DagsterInstance.ephemeral(), kwargs)
def run_command():
with capture_interrupts():
with DagsterInstance.get() as instance:
if instance.is_ephemeral:
raise Exception(
"dagster-daemon can't run using an in-memory instance. Make sure "
"the DAGSTER_HOME environment variable has been set correctly and that "
"you have created a dagster.yaml file there.")
with daemon_controller_from_instance(instance) as controller:
controller.check_daemon_loop()
def test_capture_interrupt():
outer_interrupt = False
inner_interrupt = False
with capture_interrupts():
try:
_send_interrupt_to_self()
except: # pylint: disable=bare-except
inner_interrupt = True
assert not inner_interrupt
# Verify standard interrupt handler is restored
standard_interrupt = False
try:
_send_interrupt_to_self()
except KeyboardInterrupt:
standard_interrupt = True
assert standard_interrupt
outer_interrupt = False
inner_interrupt = False
# No exception if no signal thrown
try:
with capture_interrupts():
try:
time.sleep(5)
except: # pylint: disable=bare-except
inner_interrupt = True
except: # pylint: disable=bare-except
outer_interrupt = True
assert not outer_interrupt
assert not inner_interrupt
def test_calling_raise_execution_interrupts_also_raises_any_captured_interrupts():
interrupt_from_raise_execution_interrupts = False
interrupt_after_delay = False
try:
with capture_interrupts():
_send_interrupt_to_self()
try:
with raise_execution_interrupts():
pass
except DagsterExecutionInterruptedError:
interrupt_from_raise_execution_interrupts = True
except:
interrupt_after_delay = True
assert interrupt_from_raise_execution_interrupts
assert not interrupt_after_delay
def test_interrupt_inside_nested_delay_and_raise():
interrupt_inside_nested_raise = False
interrupt_after_delay = False
try:
with capture_interrupts():
with raise_execution_interrupts():
try:
_send_interrupt_to_self()
except DagsterExecutionInterruptedError:
interrupt_inside_nested_raise = True
except: # pylint: disable=bare-except
interrupt_after_delay = True
assert interrupt_inside_nested_raise
assert not interrupt_after_delay
def execute_run_command(input_json):
with capture_interrupts():
args = check.inst(deserialize_json_to_dagster_namedtuple(input_json),
ExecuteRunArgs)
recon_pipeline = recon_pipeline_from_origin(args.pipeline_origin)
with (DagsterInstance.from_ref(args.instance_ref)
if args.instance_ref else DagsterInstance.get()) as instance:
buffer = []
def send_to_buffer(event):
buffer.append(serialize_dagster_namedtuple(event))
_execute_run_command_body(recon_pipeline, args.pipeline_run_id,
instance, send_to_buffer)
for line in buffer:
click.echo(line)
def test_no_interrupt_after_nested_delay_and_raise():
interrupt_inside_nested_raise = False
interrupt_after_delay = False
try:
with capture_interrupts():
with raise_execution_interrupts():
try:
time.sleep(5)
except: # pylint: disable=bare-except
interrupt_inside_nested_raise = True
_send_interrupt_to_self()
except: # pylint: disable=bare-except
interrupt_after_delay = True
assert not interrupt_inside_nested_raise
assert not interrupt_after_delay
def execute_step_command(input_json):
with capture_interrupts():
args = deserialize_as(input_json, ExecuteStepArgs)
with (DagsterInstance.from_ref(args.instance_ref)
if args.instance_ref else DagsterInstance.get()) as instance:
pipeline_run = instance.get_run_by_id(args.pipeline_run_id)
buff = []
for event in _execute_step_command_body(
args,
instance,
pipeline_run,
):
buff.append(serialize_dagster_namedtuple(event))
for line in buff:
click.echo(line)
def job_execute_command(**kwargs):
with capture_interrupts():
with get_instance_for_service("``dagster job execute``") as instance:
execute_execute_command(instance, kwargs, True)
def run_command():
with capture_interrupts():
with DagsterInstance.get() as instance:
_daemon_run_command(instance)
def pipeline_execute_command(**kwargs):
with capture_interrupts():
with DagsterInstance.get() as instance:
execute_execute_command(instance, kwargs)
for block in iter(lambda: file.read(1024), None):
if block:
print(block, end="", file=stream) # pylint: disable=print-call
else:
if pop_captured_interrupt() or (
parent_pid
and current_process_is_orphaned(parent_pid)):
return
time.sleep(POLLING_INTERVAL)
def execute_polling(args):
if not args or len(args) != 3:
return
filepath = args[0]
parent_pid = int(args[1])
ipc_output_file = args[2]
# Signal to the calling process that we have started and are
# ready to receive the signal to terminate once execution has finished
with open(ipc_output_file, "w"):
pass
tail_polling(filepath, sys.stdout, parent_pid)
if __name__ == "__main__":
with capture_interrupts():
execute_polling(sys.argv[1:])
def pipeline_execute_command(**kwargs):
with capture_interrupts():
with get_instance_for_service(
"``dagster pipeline execute``") as instance:
execute_execute_command(instance, kwargs)